logstash-lib 1.3.2

data/spec/filters/grok/timeout2.rb

@@ -0,0 +1,56 @@
+require "test_utils"
+require "grok-pure"
+require "timeout"
+
+describe "grok known timeout failures" do
+  extend LogStash::RSpec
+
+  describe "user reported timeout" do
+    config <<-'CONFIG'
+      filter {
+        grok { 
+         match  => [ "message", "%{SYSLOGBASE:ts1} \[\#\|%{TIMESTAMP_ISO8601:ts2}\|%{DATA} for %{PATH:url} = %{POSINT:delay} ms.%{GREEDYDATA}" ]
+        }
+      }
+    CONFIG
+
+    start = Time.now
+    line = 'Nov 13 19:23:34 qa-api1 glassfish: [#|2012-11-13T19:23:25.604+0000|INFO|glassfish3.1.2|com.locusenergy.platform.messages.LocusMessage|_ThreadID=59;_ThreadName=Thread-2;|API TIMER - Cache HIT user: null for /kiosks/194/energyreadings/data?tz=America/New_York&fields=kwh&type=gen&end=2012-11-13T23:59:59&start=2010-12-16T00:00:00-05:00&gran=yearly = 5 ms.|#]'
+
+    sample line do
+      duration = Time.now - start
+      insist { duration } < 0.03
+    end
+  end
+
+  describe "user reported timeout" do
+    config <<-'CONFIG'
+      filter {
+        grok { 
+          pattern => [
+            "%{DATA:http_host} %{IPORHOST:clientip} %{USER:ident} %{USER:http_auth} \[%{HTTPDATE:http_timestamp}\] \"%{WORD:http_method} %{DATA:http_request} HTTP/%{NUMBER:http_version}\" %{NUMBER:http_response_code} (?:%{NUMBER:bytes}|-) \"(?:%{URI:http_referrer}|-)\" %{QS:http_user_agent} %{QS:http_x_forwarded_for} %{USER:ssl_chiper} %{NUMBER:request_time} (?:%{DATA:gzip_ratio}|-) (?:%{DATA:upstream}|-) (?:%{NUMBER:upstream_time}|-) (?:%{WORD:geoip_country}|-)",
+            "%{DATA:http_host} %{IPORHOST:clientip} %{USER:ident} %{USER:http_auth} \[%{HTTPDATE:http_timestamp}\] \"%{WORD:http_method} %{DATA:http_request} HTTP/%{NUMBER:http_version}\" %{NUMBER:http_response_code} (?:%{NUMBER:bytes}|-) \"(?:%{URI:http_referrer}|-)\" %{QS:http_user_agent} %{QS:http_x_forwarded_for} %{USER:ssl_chiper} %{NUMBER:request_time} (?:%{DATA:gzip_ratio}|-) (?:%{DATA:upstream}|-) (?:%{NUMBER:upstream_time}|-)"
+          ]
+        }
+      }
+    CONFIG
+
+    start = Time.now
+    sample 'www.example.com 10.6.10.13 - - [09/Aug/2012:16:19:39 +0200] "GET /index.php HTTP/1.1" 403 211 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8.1.12) Gecko/20080201 Firefox/2.0.0.12" "-" - 0.019 - 10.6.10.12:81 0.002 US' do
+      duration = Time.now - start
+      insist { duration } < 1
+      reject { subject.tags }.include?("_grokparsefailure")
+      insist { subject["geoip_country"] } == ["US"]
+    end
+
+
+    sample 'www.example.com 10.6.10.13 - - [09/Aug/2012:16:19:39 +0200] "GET /index.php HTTP/1.1" 403 211 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8.1.12) Gecko/20080201 Firefox/2.0.0.12" "-" - 0.019 - 10.6.10.12:81 0.002 -' do
+      duration = Time.now - start
+      insist { duration } < 1
+      reject { subject.tags }.include?("_grokparsefailure")
+      insist { subject["geoip_country"].nil? } == true
+    end
+  end
+end
+
+__END__

data/spec/filters/grok/timeouts.rb

@@ -0,0 +1,39 @@
+require "test_utils"
+require "grok-pure"
+require "timeout"
+
+describe "grok known timeout failures" do
+  describe "more apache log timeouts" do
+    subject { Grok.new }
+    before :each do
+      patterns = Dir.glob(File.join(File.dirname(__FILE__), "../../../patterns/*"))
+      patterns.each { |path| subject.add_patterns_from_file(path) }
+      subject.add_pattern("RESPONSE_BYTES", '[0-9_-]+')
+      subject.add_pattern("POST_CONN_STATUS", '[+-X]')
+      subject.add_pattern("CUSTOMAPACHELOG", '%{HOSTNAME:servername} %{IP:remote_host} %{NONNEGINT:servetime_secs} \[%{HTTPDATE:apache_timestamp}\] \"%{WORD:method} %{URIPATH:url}\" %{QS:query} %{POSINT:status} %{RESPONSE_BYTES:bytes} %{POST_CONN_STATUS:post_conn_status} \"(?:%{URI:referrer}|-)\" %{QS:agent} %{QS:cookie}')
+      subject.compile("%{CUSTOMAPACHELOG}")
+    end
+
+    it "should not timeout" do
+      data = File.open(__FILE__); data.each { |line| break if line == "__END__\n" }
+      puts subject.expanded_pattern
+      data.each do |line|
+        # This timeout will toss an exception if it takes too long.
+        Timeout.timeout(1) do
+          puts :matched => subject.match(line.chomp)
+        end
+      end
+    end
+  end
+end
+
+__END__
+example.com 11.22.33.44 1 [24/Oct/2012:08:47:14 +0200] "GET /need-russia-foo.php" "?login=100536&gclid=CJHRrKeHmbMCFWbKtAodn24ArA" 200 145492 + "http://www.google.de/#hl=de&sclient=psy-ab&q=foo+russland+kosten&oq=foo+rus&gs_l=hp.1.1.0l4.0.0.1.297.0.0.0.0.0.0.0.0..0.0...0.0...1c.uULgSTd5tzc&pbx=1&bav=on.2,or.r_gc.r_pw.r_qf.&fp=c4affe4f526437d4&bpcl=35466521&biw=1600&bih=799" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4" "Cookie: -" "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" "Lang: de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" "Encoding: gzip,deflate,sdch" "Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3"
+example.com 11.22.33.44 0 [24/Oct/2012:08:47:18 +0200] "GET /ajax/ajax.fillPurposes2.php" "?destination=RUS&nationality=DEU&state=&localCode=DE" 200 271 + "http://example.com/need-russia-foo.php?login=100536&gclid=CJHRrKeHmbMCFWbKtAodn24ArA" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4" "Cookie: PHPSESSID=b9bctuvsf2bdjcorbpogdkbll6p4p8uj; code=100536; __utma=256812172.1575809976.1351061239.1351061239.1351061239.1; __utmb=256812172.2.10.1351061239; __utmc=256812172; __utmz=256812172.1351061239.1.1.utmgclid=CJHRrKeHmbMCFWbKtAodn24ArA|utmccn=(not%20set)|utmcmd=(not%20set)|utmctr=foo%20russland%20kosten; __utmv=256812172.100536" "Accept: application/json, text/javascript, */*; q=0.01" "Lang: de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" "Encoding: gzip,deflate,sdch" "Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3"
+example.com 11.22.33.44 0 [24/Oct/2012:08:49:30 +0200] "GET /ajax/ajax.fillPurposes2.php" "?destination=RUS&nationality=DEU&state=Baden-W%C3%BCrttemberg&localCode=DE" 200 271 + "http://example.com/need-russia-foo.php?login=100536&gclid=CJHRrKeHmbMCFWbKtAodn24ArA" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4" "Cookie: PHPSESSID=b9bctuvsf2bdjcorbpogdkbll6p4p8uj; code=100536; __utma=256812172.1575809976.1351061239.1351061239.1351061239.1; __utmb=256812172.2.10.1351061239; __utmc=256812172; __utmz=256812172.1351061239.1.1.utmgclid=CJHRrKeHmbMCFWbKtAodn24ArA|utmccn=(not%20set)|utmcmd=(not%20set)|utmctr=foo%20russland%20kosten; __utmv=256812172.100536" "Accept: application/json, text/javascript, */*; q=0.01" "Lang: de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" "Encoding: gzip,deflate,sdch" "Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3"
+example.com 11.22.33.44 0 [24/Oct/2012:08:49:41 +0200] "GET /ajax/ajax.fooPopup.php" "?passport_from=DEU&state_of_residence=Baden-W%C3%BCrttemberg&traveling_to[0]=RUS&traveling_for%5B0%5D=P&traveling_to%5B1%5D=&traveling_for%5B1%5D=&traveling_to%5B2%5D=&traveling_for%5B2%5D=&traveling_to%5B3%5D=&traveling_for%5B3%5D=&account_number=100536&account_exists=N" 200 2459 + "http://example.com/need-russia-foo.php?login=100536&gclid=CJHRrKeHmbMCFWbKtAodn24ArA" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4" "Cookie: PHPSESSID=b9bctuvsf2bdjcorbpogdkbll6p4p8uj; code=100536; __utma=256812172.1575809976.1351061239.1351061239.1351061239.1; __utmb=256812172.2.10.1351061239; __utmc=256812172; __utmz=256812172.1351061239.1.1.utmgclid=CJHRrKeHmbMCFWbKtAodn24ArA|utmccn=(not%20set)|utmcmd=(not%20set)|utmctr=foo%20russland%20kosten; __utmv=256812172.100536" "Accept: text/html, */*; q=0.01" "Lang: de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" "Encoding: gzip,deflate,sdch" "Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3"
+example.com 11.22.33.44 0 [24/Oct/2012:08:49:45 +0200] "GET /ajax/ajax.fooPopupValid.php" "?codeId=1588744&countryCode=RUS&travelingFor=P&entry=S&passportFrom=DEU&_=1351061386611" 200 490 + "http://example.com/need-russia-foo.php?login=100536&gclid=CJHRrKeHmbMCFWbKtAodn24ArA" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4" "Cookie: PHPSESSID=b9bctuvsf2bdjcorbpogdkbll6p4p8uj; code=100536; __utma=256812172.1575809976.1351061239.1351061239.1351061239.1; __utmb=256812172.2.10.1351061239; __utmc=256812172; __utmz=256812172.1351061239.1.1.utmgclid=CJHRrKeHmbMCFWbKtAodn24ArA|utmccn=(not%20set)|utmcmd=(not%20set)|utmctr=foo%20russland%20kosten; __utmv=256812172.100536" "Accept: text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01" "Lang: de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" "Encoding: gzip,deflate,sdch" "Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3"
+example.com 11.22.33.44 0 [24/Oct/2012:08:49:50 +0200] "POST /ajax/ajax.fooPopup.php" "" 302 - + "http://example.com/need-russia-foo.php?login=100536&gclid=CJHRrKeHmbMCFWbKtAodn24ArA" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4" "Cookie: PHPSESSID=b9bctuvsf2bdjcorbpogdkbll6p4p8uj; code=100536; __utma=256812172.1575809976.1351061239.1351061239.1351061239.1; __utmb=256812172.2.10.1351061239; __utmc=256812172; __utmz=256812172.1351061239.1.1.utmgclid=CJHRrKeHmbMCFWbKtAodn24ArA|utmccn=(not%20set)|utmcmd=(not%20set)|utmctr=foo%20russland%20kosten; __utmv=256812172.100536" "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" "Lang: de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" "Encoding: gzip,deflate,sdch" "Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3"
+example.com 11.22.33.44 1 [24/Oct/2012:08:49:50 +0200] "GET /requirements.php" "" 200 23780 + "http://example.com/need-russia-foo.php?login=100536&gclid=CJHRrKeHmbMCFWbKtAodn24ArA" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4" "Cookie: PHPSESSID=b9bctuvsf2bdjcorbpogdkbll6p4p8uj; code=100536; __utma=256812172.1575809976.1351061239.1351061239.1351061239.1; __utmb=256812172.2.10.1351061239; __utmc=256812172; __utmz=256812172.1351061239.1.1.utmgclid=CJHRrKeHmbMCFWbKtAodn24ArA|utmccn=(not%20set)|utmcmd=(not%20set)|utmctr=foo%20russland%20kosten; __utmv=256812172.100536" "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" "Lang: de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" "Encoding: gzip,deflate,sdch" "Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3"
+example.com 11.22.33.44 1 [24/Oct/2012:08:49:52 +0200] "GET /ajax/ajax.requirementsFeesTable.php" "?text=RUS_0" 200 1406 + "http://example.com/requirements.php" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4" "Cookie: PHPSESSID=b9bctuvsf2bdjcorbpogdkbll6p4p8uj; code=100536; __utma=256812172.1575809976.1351061239.1351061239.1351061239.1; __utmb=256812172.2.10.1351061239; __utmc=256812172; __utmz=256812172.1351061239.1.1.utmgclid=CJHRrKeHmbMCFWbKtAodn24ArA|utmccn=(not%20set)|utmcmd=(not%20set)|utmctr=foo%20russland%20kosten; __utmv=256812172.100536; 100536-AB-/eta-requirements=%2Feta-requirements; 100536-AB-/esta-requirements=%2Festa-requirements" "Accept: */*" "Lang: de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" "Encoding: gzip,deflate,sdch" "Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3"
+example.com 194.127.209.156 0 [24/Oct/2012:10:54:06 +0200] "GET /ajax/ajax.fillPurposes2.php" "?destination=SAU&nationality=AUT&state=&localCode=DE" 200 236 + "http://example.com/foo.php?traveling_to=SAU&traveling_for=B&nationality=AUT&login=524412&state_of_residence=Baden-W\xc3\xbcrttemberg&use_lang=de&utm_source=trans&utm_medium=email&utm_campaign=cir-foo" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)" "Cookie: PHPSESSID=h9gbvo04nomip8ca5ljko3cprrkttq9 ; code=524412" "Accept: application/json, text/javascript, */*; q=0.01" "Lang: de" "Encoding: gzip, deflate" "Charset: -"

data/spec/filters/i18n.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+require "test_utils"
+require "logstash/filters/i18n"
+
+describe LogStash::Filters::I18n do
+  extend LogStash::RSpec
+
+  describe "transliterate" do
+    config <<-CONFIG
+      filter {
+        i18n {
+          transliterate => [ "transliterateme" ]
+        }
+      }
+    CONFIG
+
+    event = {
+      "transliterateme" => [ "Ærøskøbing" ]
+    }
+
+    sample event do
+      insist { subject["transliterateme"] } == [ "AEroskobing" ]
+    end
+  end
+end

data/spec/filters/json.rb

@@ -0,0 +1,72 @@
+require "test_utils"
+require "logstash/filters/json"
+
+describe LogStash::Filters::Json do
+  extend LogStash::RSpec
+
+  describe "parse message into the event" do
+    config <<-CONFIG
+      filter {
+        json {
+          # Parse message as JSON, store the results in the 'data' field'
+          source => "message"
+        }
+      }
+    CONFIG
+
+    sample '{ "hello": "world", "list": [ 1, 2, 3 ], "hash": { "k": "v" } }' do
+      insist { subject["hello"] } == "world"
+      insist { subject["list" ] } == [1,2,3]
+      insist { subject["hash"] } == { "k" => "v" }
+    end
+  end
+
+  describe "parse message into a target field" do
+    config <<-CONFIG
+      filter {
+        json {
+          # Parse message as JSON, store the results in the 'data' field'
+          source => "message"
+          target => "data"
+        }
+      }
+    CONFIG
+
+    sample '{ "hello": "world", "list": [ 1, 2, 3 ], "hash": { "k": "v" } }' do
+      insist { subject["data"]["hello"] } == "world"
+      insist { subject["data"]["list" ] } == [1,2,3]
+      insist { subject["data"]["hash"] } == { "k" => "v" }
+    end
+  end
+
+  describe "tag invalid json" do
+    config <<-CONFIG
+      filter {
+        json {
+          # Parse message as JSON, store the results in the 'data' field'
+          source => "message"
+          target => "data"
+        }
+      }
+    CONFIG
+
+    sample "invalid json" do
+      insist { subject["tags"] }.include?("_jsonparsefailure")
+    end
+  end
+
+  describe "fixing @timestamp (#pull 733)" do
+    config <<-CONFIG
+      filter {
+        json {
+          source => "message"
+        }
+      }
+    CONFIG
+
+    sample "{ \"@timestamp\": \"2013-10-19T00:14:32.996Z\" }" do
+      insist { subject["@timestamp"] }.is_a?(Time)
+      insist { subject["@timestamp"].to_json } == "\"2013-10-19T00:14:32.996Z\""
+    end
+  end
+end

data/spec/filters/json_encode.rb

@@ -0,0 +1,37 @@
+require "test_utils"
+require "logstash/filters/json_encode"
+
+describe LogStash::Filters::JSONEncode do
+  extend LogStash::RSpec
+
+  describe "encode a field as json" do
+    config <<-CONFIG
+      filter {
+        json_encode {
+          source => "hello"
+          target => "fancy"
+        }
+      }
+    CONFIG
+
+    hash = { "hello" => { "whoa" => [ 1,2,3 ] } }
+    sample(hash) do
+      insist { JSON.parse(subject["fancy"]).to_json } == hash["hello"].to_json
+    end
+  end
+
+  describe "encode a field as json and overwrite the original" do
+    config <<-CONFIG
+      filter {
+        json_encode {
+          source => "hello"
+        }
+      }
+    CONFIG
+
+    hash = { "hello" => { "whoa" => [ 1,2,3 ] } }
+    sample(hash) do
+      insist { JSON.parse(subject["hello"]).to_json } == hash["hello"].to_json
+    end
+  end
+end

data/spec/filters/kv.rb

@@ -0,0 +1,403 @@
+require "test_utils"
+require "logstash/filters/kv"
+
+describe LogStash::Filters::KV do
+  extend LogStash::RSpec
+
+  describe "defaults" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        kv { }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["hello"] } == "world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz"] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+    end
+
+  end
+
+   describe "LOGSTASH-624: allow escaped space in key or value " do
+    config <<-CONFIG
+      filter {
+        kv { value_split => ':' }
+      }
+    CONFIG
+
+    sample 'IKE:=Quick\ Mode\ completion IKE\ IDs:=subnet:\ x.x.x.x\ (mask=\ 255.255.255.254)\ and\ host:\ y.y.y.y' do
+      insist { subject["IKE"] } == '=Quick\ Mode\ completion'
+      insist { subject['IKE\ IDs'] } == '=subnet:\ x.x.x.x\ (mask=\ 255.255.255.254)\ and\ host:\ y.y.y.y'
+    end
+  end
+
+  describe "test value_split" do
+    config <<-CONFIG
+      filter {
+        kv { value_split => ':' }
+      }
+    CONFIG
+
+    sample "hello:=world foo:bar baz=:fizz doublequoted:\"hello world\" singlequoted:'hello world'" do
+      insist { subject["hello"] } == "=world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz="] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+    end
+
+  end
+
+  describe "test field_split" do
+    config <<-CONFIG
+      filter {
+        kv { field_split => '?&' }
+      }
+    CONFIG
+
+    sample "?hello=world&foo=bar&baz=fizz&doublequoted=\"hello world\"&singlequoted='hello world'&ignoreme&foo12=bar12" do
+      insist { subject["hello"] } == "world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz"] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+      insist { subject["foo12"] } == "bar12"
+    end
+
+  end
+
+  describe  "delimited fields should override space default (reported by LOGSTASH-733)" do
+    config <<-CONFIG
+      filter {
+        kv { field_split => "|" }
+      }
+    CONFIG
+
+    sample "field1=test|field2=another test|field3=test3" do
+      insist { subject["field1"] } == "test"
+      insist { subject["field2"] } == "another test"
+      insist { subject["field3"] } == "test3"
+    end
+  end
+
+  describe "test prefix" do
+    config <<-CONFIG
+      filter {
+        kv { prefix => '__' }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["__hello"] } == "world"
+      insist { subject["__foo"] } == "bar"
+      insist { subject["__baz"] } == "fizz"
+      insist { subject["__doublequoted"] } == "hello world"
+      insist { subject["__singlequoted"] } == "hello world"
+    end
+
+  end
+
+  describe "speed test", :if => ENV["SPEEDTEST"] do
+    count = 10000 + rand(3000)
+    config <<-CONFIG
+      input {
+        generator {
+          count => #{count}
+          type => foo
+          message => "hello=world bar='baz fizzle'"
+        }
+      }
+
+      filter {
+        kv { }
+      }
+
+      output  {
+        null { }
+      }
+    CONFIG
+
+    agent do
+      p :duration => @duration, :rate => count/@duration
+    end
+  end
+
+  describe "add_tag" do
+    context "should activate when successful" do
+      config <<-CONFIG
+        filter {
+          kv { add_tag => "hello" }
+        }
+      CONFIG
+
+      sample "hello=world" do
+        insist { subject["hello"] } == "world"
+        insist { subject["tags"] }.include?("hello")
+      end
+    end
+    context "should not activate when failing" do
+      config <<-CONFIG
+        filter {
+          kv { add_tag => "hello" }
+        }
+      CONFIG
+
+      sample "this is not key value" do
+        insist { subject["tags"] }.nil?
+      end
+    end
+  end
+
+  describe "add_field" do
+    context "should activate when successful" do
+      config <<-CONFIG
+        filter {
+          kv { add_field => [ "whoa", "fancypants" ] }
+        }
+      CONFIG
+
+      sample "hello=world" do
+        insist { subject["hello"] } == "world"
+        insist { subject["whoa"] } == "fancypants"
+      end
+    end
+
+    context "should not activate when failing" do
+      config <<-CONFIG
+        filter {
+          kv { add_tag => "hello" }
+        }
+      CONFIG
+
+      sample "this is not key value" do
+        reject { subject["whoa"] } == "fancypants"
+      end
+    end
+  end
+
+  #New tests
+  describe "test target" do
+    config <<-CONFIG
+      filter {
+        kv { target => 'kv' }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["kv"]["hello"] } == "world"
+      insist { subject["kv"]["foo"] } == "bar"
+      insist { subject["kv"]["baz"] } == "fizz"
+      insist { subject["kv"]["doublequoted"] } == "hello world"
+      insist { subject["kv"]["singlequoted"] } == "hello world"
+      insist {subject["kv"].count } == 5
+    end
+
+  end
+
+  describe "test empty target" do
+    config <<-CONFIG
+      filter {
+        kv { target => 'kv' }
+      }
+    CONFIG
+
+    sample "hello:world:foo:bar:baz:fizz" do
+      insist { subject["kv"] } == nil
+    end
+  end
+
+
+  describe "test data from specific sub source" do
+    config <<-CONFIG
+      filter {
+        kv {
+          source => "data"
+        }
+      }
+    CONFIG
+    sample("data" => "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'") do
+      insist { subject["hello"] } == "world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz"] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+    end
+  end
+
+  describe "test data from specific top source" do
+    config <<-CONFIG
+      filter {
+        kv {
+          source => "@data"
+        }
+      }
+    CONFIG
+    sample({"@data" => "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'"}) do
+      insist { subject["hello"] } == "world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz"] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+    end
+  end
+
+
+  describe "test data from specific sub source and target" do
+    config <<-CONFIG
+      filter {
+        kv {
+          source => "data"
+          target => "kv"
+        }
+      }
+    CONFIG
+    sample("data" => "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'") do
+      insist { subject["kv"]["hello"] } == "world"
+      insist { subject["kv"]["foo"] } == "bar"
+      insist { subject["kv"]["baz"] } == "fizz"
+      insist { subject["kv"]["doublequoted"] } == "hello world"
+      insist { subject["kv"]["singlequoted"] } == "hello world"
+      insist { subject["kv"].count } == 5
+    end
+  end
+
+  describe "test data from nil sub source, should not issue a warning" do
+    config <<-CONFIG
+      filter {
+        kv {
+          source => "non-exisiting-field"
+          target => "kv"
+        }
+      }
+    CONFIG
+    sample "" do
+      insist { subject["non-exisiting-field"] } == nil
+      insist { subject["kv"] } == nil
+    end
+  end
+
+  describe "test include_keys" do
+    config <<-CONFIG
+      filter {
+        kv {
+          include_keys => [ "foo", "singlequoted" ]
+        }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["foo"] } == "bar"
+      insist { subject["singlequoted"] } == "hello world"
+    end
+  end
+
+  describe "test exclude_keys" do
+    config <<-CONFIG
+      filter {
+        kv {
+          exclude_keys => [ "foo", "singlequoted" ]
+        }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["hello"] } == "world"
+      insist { subject["baz"] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+    end
+  end
+
+  describe "test include_keys with prefix" do
+    config <<-CONFIG
+      filter {
+        kv {
+          include_keys => [ "foo", "singlequoted" ]
+          prefix       => "__"
+        }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["__foo"] } == "bar"
+      insist { subject["__singlequoted"] } == "hello world"
+    end
+  end
+
+  describe "test exclude_keys with prefix" do
+    config <<-CONFIG
+      filter {
+        kv {
+          exclude_keys => [ "foo", "singlequoted" ]
+          prefix       => "__"
+        }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["__hello"] } == "world"
+      insist { subject["__baz"] } == "fizz"
+      insist { subject["__doublequoted"] } == "hello world"
+    end
+  end
+
+  describe "test include_keys and exclude_keys" do
+    config <<-CONFIG
+      filter {
+        kv {
+          # This should exclude everything as a result of both settings.
+          include_keys => [ "foo", "singlequoted" ]
+          exclude_keys => [ "foo", "singlequoted" ]
+        }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      %w(hello foo baz doublequoted singlequoted).each do |field|
+        reject { subject }.include?(field)
+      end
+    end
+  end
+
+  describe "test default_keys" do
+    config <<-CONFIG
+      filter {
+        kv {
+          default_keys => [ "foo", "xxx",
+                            "goo", "yyy" ]
+        }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["hello"] } == "world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["goo"] } == "yyy"
+      insist { subject["baz"] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+    end
+  end
+
+  describe "overwriting a string field (often the source)" do
+    config <<-CONFIG
+      filter {
+        kv {
+          source => "happy"
+          target => "happy"
+        }
+      }
+    CONFIG
+
+    sample("happy" => "foo=bar baz=fizz") do
+      insist { subject["[happy][foo]"] } == "bar"
+      insist { subject["[happy][baz]"] } == "fizz"
+    end
+
+  end
+
+end