logstash-lib 1.3.2

data/spec/config.rb

@@ -0,0 +1,31 @@
+# config syntax tests
+#
+
+require "logstash/config/grammar"
+require "logstash/config/config_ast"
+
+describe LogStashConfigParser do
+  it "should permit single-quoted attribute names" do
+    parser = LogStashConfigParser.new
+    config = parser.parse(%q(
+      input {
+        example {
+          'foo' => 'bar'
+          test => { 'bar' => 'baz' }
+        }
+      }
+    ))
+
+    reject { config }.nil?
+  end
+
+  it "should permit empty plugin sections" do
+    parser = LogStashConfigParser.new
+    config = parser.parse(%q(
+      filter {
+      }
+    ))
+
+    reject { config }.nil?
+  end
+end

data/spec/event.rb

@@ -0,0 +1,165 @@
+require "logstash/event"
+require "insist"
+
+describe LogStash::Event do
+  subject do
+    LogStash::Event.new(
+      "@timestamp" => Time.iso8601("2013-01-01T00:00:00.000Z"),
+      "type" => "sprintf",
+      "message" => "hello world",
+      "tags" => [ "tag1" ],
+      "source" => "/home/foo",
+      "a" => "b", 
+      "c" => {
+        "d" => "f",
+        "e" => {"f" => "g"}
+      }, 
+      "f" => { "g" => { "h" => "i" } },
+      "j" => { 
+          "k1" => "v", 
+          "k2" => [ "w", "x" ],
+          "k3" => {"4" => "m"},
+          5 => 6,
+          "5" => 7
+      } 
+    )
+  end
+
+  context "[]=" do
+    it "should raise an exception if you attempt to set @timestamp to a value type other than a Time object" do
+      insist { subject["@timestamp"] = "crash!" }.raises(TypeError)
+    end
+  end
+
+  context "#sprintf" do
+    it "should report a unix timestamp for %{+%s}" do
+      insist { subject.sprintf("%{+%s}") } == "1356998400"
+    end
+    
+    it "should report a time with %{+format} syntax", :if => RUBY_ENGINE == "jruby" do
+      insist { subject.sprintf("%{+YYYY}") } == "2013"
+      insist { subject.sprintf("%{+MM}") } == "01"
+      insist { subject.sprintf("%{+HH}") } == "00"
+    end
+  
+    it "should report fields with %{field} syntax" do
+      insist { subject.sprintf("%{type}") } == "sprintf"
+      insist { subject.sprintf("%{message}") } == subject["message"]
+    end
+    
+    it "should print deep fields" do
+      insist { subject.sprintf("%{[j][k1]}") } == "v"
+      insist { subject.sprintf("%{[j][k2][0]}") } == "w"
+    end
+
+    it "should be able to take a non-string for the format" do
+      insist { subject.sprintf(2) } == "2"
+    end
+  end
+  
+  context "#[]" do
+    it "should fetch data" do
+      insist { subject["type"] } == "sprintf"
+    end
+    it "should fetch fields" do
+      insist { subject["a"] } == "b"
+      insist { subject['c']['d'] } == "f"
+    end
+    it "should fetch deep fields" do
+      insist { subject["[j][k1]"] } == "v"
+      insist { subject["[c][d]"] } == "f"
+      insist { subject['[f][g][h]'] } == "i"
+      insist { subject['[j][k3][4]'] } == "m"
+      insist { subject['[j][5]'] } == 7
+
+    end
+
+    it "should be fast?", :if => ENV["SPEEDTEST"] do
+      2.times do
+        start = Time.now
+        100000.times { subject["[j][k1]"] }
+        puts "Duration: #{Time.now - start}"
+      end
+    end
+  end
+
+  context "#append" do
+    it "should append strings to an array" do
+      subject.append(LogStash::Event.new("message" => "another thing"))
+      insist { subject["message"] } == [ "hello world", "another thing" ]
+    end
+  
+    it "should concatenate tags" do
+      subject.append(LogStash::Event.new("tags" => [ "tag2" ]))
+      insist { subject["tags"] } == [ "tag1", "tag2" ]
+    end
+  
+    context "when event field is nil" do
+      it "should add single value as string" do
+        subject.append(LogStash::Event.new({"field1" => "append1"}))
+        insist { subject[ "field1" ] } == "append1"
+      end
+      it "should add multi values as array" do
+        subject.append(LogStash::Event.new({"field1" => [ "append1","append2" ]}))
+        insist { subject[ "field1" ] } == [ "append1","append2" ]
+      end
+    end
+  
+    context "when event field is a string" do
+      before { subject[ "field1" ] = "original1" }
+  
+      it "should append string to values, if different from current" do
+        subject.append(LogStash::Event.new({"field1" => "append1"}))
+        insist { subject[ "field1" ] } == [ "original1", "append1" ]
+      end
+      it "should not change value, if appended value is equal current" do
+        subject.append(LogStash::Event.new({"field1" => "original1"}))
+        insist { subject[ "field1" ] } == "original1"
+      end
+      it "should concatenate values in an array" do
+        subject.append(LogStash::Event.new({"field1" => [ "append1" ]}))
+        insist { subject[ "field1" ] } == [ "original1", "append1" ]
+      end
+      it "should join array, removing duplicates" do
+        subject.append(LogStash::Event.new({"field1" => [ "append1","original1" ]}))
+        insist { subject[ "field1" ] } == [ "original1", "append1" ]
+      end
+    end
+    context "when event field is an array" do
+      before { subject[ "field1" ] = [ "original1", "original2" ] }
+  
+      it "should append string values to array, if not present in array" do
+        subject.append(LogStash::Event.new({"field1" => "append1"}))
+        insist { subject[ "field1" ] } == [ "original1", "original2", "append1" ]
+      end
+      it "should not append string values, if the array already contains it" do
+        subject.append(LogStash::Event.new({"field1" => "original1"}))
+        insist { subject[ "field1" ] } == [ "original1", "original2" ]
+      end
+      it "should join array, removing duplicates" do
+        subject.append(LogStash::Event.new({"field1" => [ "append1","original1" ]}))
+        insist { subject[ "field1" ] } == [ "original1", "original2", "append1" ]
+      end
+    end
+  end
+
+  it "timestamp parsing speed", :if => ENV["SPEEDTEST"] do
+    warmup = 10000
+    count = 1000000
+
+    data = { "@timestamp" => "2013-12-21T07:25:06.605Z" }
+    event = LogStash::Event.new(data)
+    insist { event["@timestamp"] }.is_a?(Time)
+
+    duration = 0
+    [warmup, count].each do |i|
+      start = Time.now
+      i.times do
+        data = { "@timestamp" => "2013-12-21T07:25:06.605Z" }
+        LogStash::Event.new(data.clone)
+      end
+      duration = Time.now - start
+    end
+    puts "event @timestamp parse rate: #{count / duration}/sec"
+  end
+end

data/spec/examples/fail2ban.rb

@@ -0,0 +1,28 @@
+require "test_utils"
+
+describe "fail2ban logs", :if => RUBY_ENGINE == "jruby"  do
+  extend LogStash::RSpec
+
+  # The logstash config goes here.
+  # At this time, only filters are supported.
+  config <<-CONFIG
+    filter {
+      grok {
+        pattern => "^%{TIMESTAMP_ISO8601:timestamp} fail2ban\.actions: %{WORD:level} \\[%{WORD:program}\\] %{WORD:action} %{IP:ip}"
+        singles => true
+      }
+      date {
+        match => [ "timestamp", "yyyy-MM-dd HH:mm:ss,SSS" ]
+      }
+      mutate {
+        remove => timestamp
+      }
+    }
+  CONFIG
+
+  sample "2013-06-28 15:10:59,891 fail2ban.actions: WARNING [ssh] Ban 95.78.163.5" do
+    insist { subject["program"] } == "ssh"
+    insist { subject["action"] } == "Ban"
+    insist { subject["ip"] } == "95.78.163.5"
+  end
+end

data/spec/examples/graphite-input.rb

@@ -0,0 +1,41 @@
+require "test_utils"
+
+describe "receive graphite input", :if => RUBY_ENGINE == "jruby" do
+  extend LogStash::RSpec
+
+  # The logstash config goes here.
+  # At this time, only filters are supported.
+  config <<-CONFIG
+    # input {
+    #   tcp {
+    #     port => 1234
+    #     mode => server
+    #     type => graphite
+    #   }
+    # }
+    filter {
+      grok {
+        pattern => "%{DATA:name} %{NUMBER:value:float} %{POSINT:ts}"
+        singles => true
+      }
+      date {
+        match => ["ts", UNIX]
+      }
+      mutate {
+        remove => ts
+      }
+    }
+  CONFIG
+
+  type "graphite"
+
+  sample "foo.bar.baz 4025.34 1364606522" do
+    insist { subject }.include?("name")
+    insist { subject }.include?("value")
+
+    insist { subject["name"] } == "foo.bar.baz"
+    insist { subject["value"] } == 4025.34
+    insist { subject["@timestamp"] } == Time.iso8601("2013-03-30T01:22:02.000Z")
+    
+  end
+end

data/spec/examples/mysql-slow-query.rb

@@ -0,0 +1,70 @@
+require "test_utils"
+
+# Skip until we convert this to use multiline codec
+describe "parse mysql slow query log", :if => false do
+  extend LogStash::RSpec
+
+  config <<-'CONFIG'
+    filter {
+      grep {
+        # Drop the '# Time:' lines since they only appear when the 'time'
+        # changes and are omitted otherwise. Further, there's always (from what
+        # I have seen) a 'SET timestamp=123456789' line in each query event, so
+        # I use that as the timestamp instead.
+        #
+        # mysql logs are messed up, so this is pretty much best effort.
+        match => [ "@message", "^# Time: " ]
+        negate => true
+      }
+
+      grok {
+        singles => true
+        pattern => [
+          "^# User@Host: %{USER:user}\[[^\]]+\] @ %{HOST:host} \[%{IP:ip}?]",
+          "^# Query_time: %{NUMBER:duration:float} \s*Lock_time: %{NUMBER:lock_wait:float} \s*Rows_sent: %{NUMBER:results:int} \s*Rows_examined: %{NUMBER:scanned:int}",
+          "^SET timestamp=%{NUMBER:timestamp};"
+        ]
+      }
+
+      multiline {
+        pattern => "^# User@Host: "
+        negate => true
+        what => previous
+      }
+
+      date {
+        match => ["timestamp", UNIX]
+      }
+
+      mutate {
+        remove => "timestamp"
+      }
+    }
+  CONFIG
+
+  lines = <<-'MYSQL_SLOW_LOGS'
+# Time: 121004  6:00:27
+# User@Host: someuser[someuser] @ db.example.com [1.2.3.4]
+# Query_time: 0.018143  Lock_time: 0.000042 Rows_sent: 237  Rows_examined: 286
+use somedb;
+SET timestamp=1349355627;
+SELECT option_name, option_value FROM wp_options WHERE autoload = 'yes';
+MYSQL_SLOW_LOGS
+
+  sample lines.split("\n") do
+    reject { subject }.is_a? Array # 1 event expected
+    insist { subject.message.split("\n").size } == 5 # 5 lines
+
+    lines.split("\n")[1..5].each_with_index do |line, i|
+      insist { subject.message.split("\n")[i] } == line
+    end
+
+    insist { subject["user"] } == "someuser"
+    insist { subject["host"] } == "db.example.com"
+    insist { subject["ip"] } == "1.2.3.4"
+    insist { subject["duration"] } == 0.018143
+    insist { subject["lock_wait"] } == 0.000042
+    insist { subject["results"] } == 237
+    insist { subject["scanned"] } == 286
+  end
+end

data/spec/examples/parse-apache-logs.rb

@@ -0,0 +1,66 @@
+require "test_utils"
+
+describe "apache common log format", :if => RUBY_ENGINE == "jruby" do
+  extend LogStash::RSpec
+
+  # The logstash config goes here.
+  # At this time, only filters are supported.
+  config <<-CONFIG
+    filter {
+      grok {
+        pattern => "%{COMBINEDAPACHELOG}"
+        singles => true
+      }
+      date {
+        match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
+      }
+    }
+  CONFIG
+
+  # Here we provide a sample log event for the testing suite.
+  #
+  # Any filters you define above will be applied the same way the logstash
+  # agent performs. Inside the 'sample ... ' block the 'subject' will be
+  # a LogStash::Event object for you to inspect and verify for correctness.
+  sample '198.151.8.4 - - [29/Aug/2012:20:17:38 -0400] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:14.0) Gecko/20100101 Firefox/14.0.1"' do
+
+    # These 'insist' and 'reject' calls use my 'insist' rubygem.
+    # See http://rubydoc.info/gems/insist for more info.
+
+    # Require that grok does not fail to parse this event.
+    insist { subject["tags"] }.nil?
+
+    # Ensure that grok captures certain expected fields.
+    insist { subject }.include?("agent")
+    insist { subject }.include?("bytes")
+    insist { subject }.include?("clientip")
+    insist { subject }.include?("httpversion")
+    insist { subject }.include?("timestamp")
+    insist { subject }.include?("verb")
+    insist { subject }.include?("response")
+    insist { subject }.include?("request")
+
+    # Ensure that those fields match expected values from the event.
+    insist { subject["clientip"] } == "198.151.8.4"
+    insist { subject["timestamp"] } == "29/Aug/2012:20:17:38 -0400"
+    insist { subject["verb"] } == "GET"
+    insist { subject["request"] } == "/favicon.ico"
+    insist { subject["httpversion"] } == "1.1"
+    insist { subject["response"] } == "200"
+    insist { subject["bytes"] } == "3638"
+    insist { subject["referrer"] } == '"-"'
+    insist { subject["agent"] } == "\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:14.0) Gecko/20100101 Firefox/14.0.1\""
+
+    # Verify date parsing
+    insist { subject.timestamp } == Time.iso8601("2012-08-30T00:17:38.000Z")
+  end
+
+  sample '61.135.248.195 - - [26/Sep/2012:11:49:20 -0400] "GET /projects/keynav/ HTTP/1.1" 200 18985 "" "Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )"' do
+    insist { subject["tags"] }.nil?
+    insist { subject["clientip"] } == "61.135.248.195"
+  end
+
+  sample '72.14.164.185 - - [25/Sep/2012:12:05:02 -0400] "GET /robots.txt HTTP/1.1" 200 - "www.brandimensions.com" "BDFetch"' do
+    insist { subject["tags"] }.nil?
+  end
+end

data/spec/examples/parse-haproxy-logs.rb

@@ -0,0 +1,115 @@
+require "test_utils"
+
+describe "haproxy httplog format" do
+  extend LogStash::RSpec
+
+  # The logstash config goes here.
+  # At this time, only filters are supported.
+  config <<-CONFIG
+  filter {
+    grok {
+      pattern => "%{HAPROXYHTTP}"
+    }
+  }
+  CONFIG
+  # Here we provide a sample log event for the testing suite.
+  #
+  # Any filters you define above will be applied the same way the logstash
+  # agent performs. Inside the 'sample ... ' block the 'subject' will be
+  # a LogStash::Event object for you to inspect and verify for correctness.
+  # HAPROXYHTTP %{SYSLOGTIMESTAMP:syslog_timestamp} %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"
+
+  sample 'Feb  6 12:14:14 localhost haproxy[14389]: 10.0.1.2:33317 [06/Feb/2009:12:14:14.655] http-in static/srv1 10/0/30/69/109 200 2750 - - ---- 1/1/1/1/0 0/0 {1wt.eu} {} "GET /index.html HTTP/1.1"' do
+
+    # These 'insist' and 'reject' calls use my 'insist' rubygem.
+    # See http://rubydoc.info/gems/insist for more info.
+
+    # Require that grok does not fail to parse this event.
+    insist { subject["tags"] }.nil?
+
+
+    # Ensure that grok captures certain expected fields.
+    insist { subject }.include?("syslog_timestamp")
+    insist { subject }.include?("syslog_server")
+    insist { subject }.include?("program")
+    insist { subject }.include?("pid")
+    insist { subject }.include?("client_ip")
+    insist { subject }.include?("client_port")
+    insist { subject }.include?("accept_date")
+    insist { subject }.include?("haproxy_monthday")
+    insist { subject }.include?("haproxy_month")
+    insist { subject }.include?("haproxy_year")
+    insist { subject }.include?("haproxy_time")
+    insist { subject }.include?("haproxy_hour")
+    insist { subject }.include?("haproxy_minute")
+    insist { subject }.include?("haproxy_second")
+    insist { subject }.include?("haproxy_milliseconds")
+    insist { subject }.include?("frontend_name")
+    insist { subject }.include?("backend_name")
+    insist { subject }.include?("server_name")
+    insist { subject }.include?("time_request")
+    insist { subject }.include?("time_queue")
+    insist { subject }.include?("time_backend_connect")
+    insist { subject }.include?("time_backend_response")
+    insist { subject }.include?("time_duration")
+    insist { subject }.include?("http_status_code")
+    insist { subject }.include?("bytes_read")
+    insist { subject }.include?("captured_request_cookie")
+    insist { subject }.include?("captured_response_cookie")
+    insist { subject }.include?("termination_state")
+    insist { subject }.include?("actconn")
+    insist { subject }.include?("feconn")
+    insist { subject }.include?("beconn")
+    insist { subject }.include?("srvconn")
+    insist { subject }.include?("retries")
+    insist { subject }.include?("srv_queue")
+    insist { subject }.include?("backend_queue")
+    insist { subject }.include?("captured_request_headers")
+    insist { subject }.include?("http_verb")
+    insist { subject }.include?("http_request")
+    insist { subject }.include?("http_version")
+
+#    # Ensure that those fields match expected values from the event.
+
+    insist{ subject["syslog_timestamp"] } == "Feb  6 12:14:14"
+    insist{ subject["syslog_server"] } == "localhost"
+    insist{ subject["program"] } == "haproxy"
+    insist{ subject["pid"] } == "14389"
+    insist{ subject["client_ip"] } == "10.0.1.2"
+    insist{ subject["client_port"] } == "33317"
+    insist{ subject["accept_date"] } == "06/Feb/2009:12:14:14.655"
+    insist{ subject["haproxy_monthday"] } == "06"
+    insist{ subject["haproxy_month"] } == "Feb"
+    insist{ subject["haproxy_year"] } == "2009"
+    insist{ subject["haproxy_time"] } == "12:14:14"
+    insist{ subject["haproxy_hour"] } == "12"
+    insist{ subject["haproxy_minute"] } == "14"
+    insist{ subject["haproxy_second"] } == "14"
+    insist{ subject["haproxy_milliseconds"] } == "655"
+    insist{ subject["frontend_name"] } == "http-in"
+    insist{ subject["backend_name"] } == "static"
+    insist{ subject["server_name"] } == "srv1"
+    insist{ subject["time_request"] } == "10"
+    insist{ subject["time_queue"] } == "0"
+    insist{ subject["time_backend_connect"] } == "30"
+    insist{ subject["time_backend_response"] } == "69"
+    insist{ subject["time_duration"] } == "109"
+    insist{ subject["http_status_code"] } == "200"
+    insist{ subject["bytes_read"] } == "2750"
+    insist{ subject["captured_request_cookie"] } == "-"
+    insist{ subject["captured_response_cookie"] } == "-"
+    insist{ subject["termination_state"] } == "----"
+    insist{ subject["actconn"] } == "1"
+    insist{ subject["feconn"] } == "1"
+    insist{ subject["beconn"] } == "1"
+    insist{ subject["srvconn"] } == "1"
+    insist{ subject["retries"] } == "0"
+    insist{ subject["srv_queue"] } == "0"
+    insist{ subject["backend_queue"] } == "0"
+    insist{ subject["captured_request_headers"] } == "1wt.eu"
+    insist{ subject["http_verb"] } == "GET"
+    insist{ subject["http_request"] } == "/index.html"
+    insist{ subject["http_version"] } == "1.1"
+  end
+
+end