logstash-lib 1.3.2

data/spec/filters/csv.rb

@@ -0,0 +1,174 @@
+require "test_utils"
+require "logstash/filters/csv"
+
+describe LogStash::Filters::CSV do
+  extend LogStash::RSpec
+
+  describe "all defaults" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        csv { }
+      }
+    CONFIG
+
+    sample "big,bird,sesame street" do
+      insist { subject["column1"] } == "big"
+      insist { subject["column2"] } == "bird"
+      insist { subject["column3"] } == "sesame street"
+    end
+  end
+
+  describe "custom separator" do
+    config <<-CONFIG
+      filter {
+        csv {
+          separator => ";"
+        }
+      }
+    CONFIG
+
+    sample "big,bird;sesame street" do
+      insist { subject["column1"] } == "big,bird"
+      insist { subject["column2"] } == "sesame street"
+    end
+  end
+
+  describe "custom quote char" do
+    config <<-CONFIG
+      filter {
+        csv {
+          quote_char => "'"
+        }
+      }
+    CONFIG
+
+    sample "big,bird,'sesame street'" do
+      insist { subject["column1"] } == "big"
+      insist { subject["column2"] } == "bird"
+      insist { subject["column3"] } == "sesame street"
+    end
+  end
+
+  describe "default quote char" do
+    config <<-CONFIG
+      filter {
+        csv {
+        }
+      }
+    CONFIG
+
+    sample 'big,bird,"sesame, street"' do
+      insist { subject["column1"] } == "big"
+      insist { subject["column2"] } == "bird"
+      insist { subject["column3"] } == "sesame, street"
+    end
+  end
+  describe "null quote char" do
+    config <<-CONFIG
+      filter {
+        csv {
+          quote_char => "\x00"
+        }
+      }
+    CONFIG
+
+    sample 'big,bird,"sesame" street' do
+      insist { subject["column1"] } == 'big'
+      insist { subject["column2"] } == 'bird'
+      insist { subject["column3"] } == '"sesame" street'
+    end
+  end
+
+  describe "given columns" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        csv {
+          columns => ["first", "last", "address" ]
+        }
+      }
+    CONFIG
+
+    sample "big,bird,sesame street" do
+      insist { subject["first"] } == "big"
+      insist { subject["last"] } == "bird"
+      insist { subject["address"] } == "sesame street"
+    end
+  end
+
+  describe "parse csv with more data than defined column names" do
+    config <<-CONFIG
+      filter {
+        csv {
+          columns => ["custom1", "custom2"]
+        }
+      }
+    CONFIG
+
+    sample "val1,val2,val3" do
+      insist { subject["custom1"] } == "val1"
+      insist { subject["custom2"] } == "val2"
+      insist { subject["column3"] } == "val3"
+    end
+  end
+
+
+  describe "parse csv from a given source with column names" do
+    config <<-CONFIG
+      filter {
+        csv {
+          source => "datafield"
+          columns => ["custom1", "custom2", "custom3"]
+        }
+      }
+    CONFIG
+
+    sample("datafield" => "val1,val2,val3") do
+      insist { subject["custom1"] } == "val1"
+      insist { subject["custom2"] } == "val2"
+      insist { subject["custom3"] } == "val3"
+    end
+  end
+
+  describe "given target" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        csv {
+          target => "data"
+        }
+      }
+    CONFIG
+
+    sample "big,bird,sesame street" do
+      insist { subject["data"]["column1"] } == "big"
+      insist { subject["data"]["column2"] } == "bird"
+      insist { subject["data"]["column3"] } == "sesame street"
+    end
+  end
+
+  describe "given target and source" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        csv {
+          source => "datain"
+          target => "data"
+        }
+      }
+    CONFIG
+
+    sample("datain" => "big,bird,sesame street") do
+      insist { subject["data"]["column1"] } == "big"
+      insist { subject["data"]["column2"] } == "bird"
+      insist { subject["data"]["column3"] } == "sesame street"
+    end
+  end
+
+
+end

data/spec/filters/date.rb

@@ -0,0 +1,285 @@
+require "test_utils"
+require "logstash/filters/date"
+
+puts "Skipping date performance tests because this ruby is not jruby" if RUBY_ENGINE != "jruby"
+RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
+  extend LogStash::RSpec
+
+  describe "giving an invalid match config, raise a configuration error" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "mydate"]
+        }
+      }
+    CONFIG
+
+    sample "not_really_important" do
+      insist {subject}.raises LogStash::ConfigurationError
+    end
+
+  end
+
+  describe "parsing with ISO8601" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "mydate", "ISO8601" ]
+        }
+      }
+    CONFIG
+
+    times = {
+      "2001-01-01T00:00:00-0800"         => "2001-01-01T08:00:00.000Z",
+      "1974-03-02T04:09:09-0800"         => "1974-03-02T12:09:09.000Z",
+      "2010-05-03T08:18:18+00:00"        => "2010-05-03T08:18:18.000Z",
+      "2004-07-04T12:27:27-00:00"        => "2004-07-04T12:27:27.000Z",
+      "2001-09-05T16:36:36+0000"         => "2001-09-05T16:36:36.000Z",
+      "2001-11-06T20:45:45-0000"         => "2001-11-06T20:45:45.000Z",
+      "2001-12-07T23:54:54Z"             => "2001-12-07T23:54:54.000Z",
+
+      # TODO: This test assumes PDT
+      #"2001-01-01T00:00:00.123"          => "2001-01-01T08:00:00.123Z",
+
+      "2010-05-03T08:18:18.123+00:00"    => "2010-05-03T08:18:18.123Z",
+      "2004-07-04T12:27:27.123-04:00"    => "2004-07-04T16:27:27.123Z",
+      "2001-09-05T16:36:36.123+0700"     => "2001-09-05T09:36:36.123Z",
+      "2001-11-06T20:45:45.123-0000"     => "2001-11-06T20:45:45.123Z",
+      "2001-12-07T23:54:54.123Z"         => "2001-12-07T23:54:54.123Z",
+    }
+
+    times.each do |input, output|
+      sample("mydate" => input) do
+        begin 
+          insist { subject["mydate"] } == input
+          insist { subject["@timestamp"] } == Time.iso8601(output).utc
+        rescue
+          #require "pry"; binding.pry
+          raise
+        end
+      end
+    end # times.each
+  end
+
+  describe "parsing with java SimpleDateFormat syntax" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "mydate", "MMM dd HH:mm:ss Z" ]
+        }
+      }
+    CONFIG
+
+    now = Time.now
+    year = now.year
+    require 'java'
+
+    times = {
+      "Nov 24 01:29:01 -0800" => "#{year}-11-24T09:29:01.000Z",
+    }
+    times.each do |input, output|
+      sample("mydate" => input) do
+        insist { subject["mydate"] } == input
+        insist { subject["@timestamp"] } == Time.iso8601(output).utc
+      end
+    end # times.each
+  end
+
+  describe "parsing with UNIX" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "mydate", "UNIX" ]
+        }
+      }
+    CONFIG
+
+    times = {
+      "0"          => "1970-01-01T00:00:00.000Z",
+      "1000000000" => "2001-09-09T01:46:40.000Z",
+
+      # LOGSTASH-279 - sometimes the field is a number.
+      0          => "1970-01-01T00:00:00.000Z",
+      1000000000 => "2001-09-09T01:46:40.000Z"
+    }
+    times.each do |input, output|
+      sample("mydate" => input) do
+        insist { subject["mydate"] } == input
+        insist { subject["@timestamp"] } == Time.iso8601(output).utc
+      end
+    end # times.each
+  end
+
+  describe "parsing microsecond-precise times with UNIX (#213)" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "mydate", "UNIX" ]
+        }
+      }
+    CONFIG
+
+    sample("mydate" => "1350414944.123456") do
+      # Joda time only supports milliseconds :\
+      insist { subject.timestamp } == Time.iso8601("2012-10-16T12:15:44.123-07:00").utc
+    end
+  end
+
+  describe "parsing with UNIX_MS" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "mydate", "UNIX_MS" ]
+        }
+      }
+    CONFIG
+
+    times = {
+      "0"          => "1970-01-01T00:00:00.000Z",
+      "456"          => "1970-01-01T00:00:00.456Z",
+      "1000000000123" => "2001-09-09T01:46:40.123Z",
+
+      # LOGSTASH-279 - sometimes the field is a number.
+      0          => "1970-01-01T00:00:00.000Z",
+      456          => "1970-01-01T00:00:00.456Z",
+      1000000000123 => "2001-09-09T01:46:40.123Z"
+    }
+    times.each do |input, output|
+      sample("mydate" => input) do
+        insist { subject["mydate"] } == input
+        insist { subject["@timestamp"] } == Time.iso8601(output)
+      end
+    end # times.each
+  end
+
+  describe "failed parses should not cause a failure (LOGSTASH-641)" do
+    config <<-'CONFIG'
+      input {
+        generator {
+          lines => [
+            '{ "mydate": "this will not parse" }',
+            '{ }'
+          ]
+          codec => json
+          type => foo
+          count => 1
+        }
+      }
+      filter {
+        date {
+          match => [ "mydate", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+        }
+      }
+      output { 
+        null { }
+      }
+    CONFIG
+
+    agent do
+      # nothing to do, if this crashes it's an error..
+    end
+  end
+
+  describe "TAI64N support" do
+    config <<-'CONFIG'
+      filter {
+        date {
+          match => [ "t",  TAI64N ]
+        }
+      }
+    CONFIG
+
+    # Try without leading "@"
+    sample("t" => "4000000050d506482dbdf024") do
+      insist { subject.timestamp } == Time.iso8601("2012-12-22T01:00:46.767Z").utc
+    end
+
+    # Should still parse successfully if it's a full tai64n time (with leading
+    # '@')
+    sample("t" => "@4000000050d506482dbdf024") do
+      insist { subject.timestamp } == Time.iso8601("2012-12-22T01:00:46.767Z").utc
+    end
+  end
+
+  describe "accept match config option with hash value (LOGSTASH-735)" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "mydate", "ISO8601" ]
+        }
+      }
+    CONFIG
+
+    time = "2001-09-09T01:46:40.000Z"
+
+    sample("mydate" => time) do
+      insist { subject["mydate"] } == time
+      insist { subject["@timestamp"] } == Time.iso8601(time).utc
+    end
+  end
+  
+  describe "support deep nested field access" do
+    config <<-CONFIG
+      filter { 
+        date {
+          match => [ "[data][deep]", "ISO8601" ]
+        }
+      }
+    CONFIG
+    
+    sample("data" => { "deep" => "2013-01-01T00:00:00.000Z" }) do
+      insist { subject["@timestamp"] } == Time.iso8601("2013-01-01T00:00:00.000Z").utc
+    end
+  end
+
+  describe "failing to parse should not throw an exception" do
+    config <<-CONFIG
+      filter { 
+        date {
+          match => [ "thedate", "yyyy/MM/dd" ]
+        }
+      }
+    CONFIG
+
+    sample("thedate" => "2013/Apr/21") do
+      insist { subject["@timestamp"] } != "2013-04-21T00:00:00.000Z"
+    end
+  end
+
+  describe "parsing with timezone parameter" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => ["mydate", "yyyy MMM dd HH:mm:ss"]
+          timezone => "America/Los_Angeles"
+        }
+      }
+    CONFIG
+
+    require 'java'
+    times = {
+      "2013 Nov 24 01:29:01" => "2013-11-24T09:29:01.000Z",
+      "2013 Jun 24 01:29:01" => "2013-06-24T08:29:01.000Z",
+    }
+    times.each do |input, output|
+      sample("mydate" => input) do
+        insist { subject["mydate"] } == input
+        insist { subject["@timestamp"] } == Time.iso8601(output).utc
+      end
+    end # times.each
+  end
+
+  describe "LOGSTASH-34 - Default year should be this year" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "message", "EEE MMM dd HH:mm:ss" ]
+        }
+      }
+    CONFIG
+
+    sample "Sun Jun 02 20:38:03" do
+      insist { subject["@timestamp"].year } == Time.now.year
+    end
+  end
+end