logstash-lib 1.3.2

data/require-analyze.rb

@@ -0,0 +1,22 @@
+require "csv"
+
+#0.003,psych/nodes/mapping,/Users/jls/.rvm/rubies/jruby-1.7.8/lib/ruby/shared/psych/nodes.rb:6:in `(root)'
+
+durations = {}
+durations.default = 0
+
+CSV.foreach(ARGV[0]) do |duration, path, source|
+  source, line, where = source.split(":")
+  #{"0.002"=>"/Users/jls/projects/logstash/vendor/bundle/jruby/1.9/gems/clamp-0.6.3/lib/clamp.rb"}
+  if source.include?("jruby/1.9/gems")
+    # Get the gem name
+    source = source.gsub(/.*\/jruby\/1.9\/gems/, "")[/[^\/]+/]
+  elsif source.include?("/lib/logstash/")
+    source = source.gsub(/^.*(\/lib\/logstash\/)/, "/lib/logstash/")
+  end
+  durations[source] += duration.to_f
+end
+
+durations.sort_by { |k,v| v }.each do |k,v| 
+  puts "#{v} #{k}"
+end

data/spec/README.md

@@ -0,0 +1,14 @@
+# How to run these tests
+
+Run one:
+
+  `rspec spec/the/test.rb`
+
+Run them all:
+
+  `rspec spec/**/*.rb`
+
+Debug one test:
+
+  `LOGSTASH_DEBUG=y rspec spec/the/test.rb`
+

data/spec/codecs/edn.rb

@@ -0,0 +1,40 @@
+require "logstash/codecs/edn"
+require "logstash/event"
+require "insist"
+require "edn"
+
+describe LogStash::Codecs::EDN do
+  subject do
+    next LogStash::Codecs::EDN.new
+  end
+
+  context "#decode" do
+    it "should return an event from edn data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a", "b", "c"]}}
+      subject.decode(data.to_edn) do |event|
+        insist { event }.is_a?(LogStash::Event)
+        insist { event["foo"] } == data["foo"]
+        insist { event["baz"] } == data["baz"]
+        insist { event["bah"] } == data["bah"]
+      end
+    end
+  end
+
+  context "#encode" do
+    it "should return edn data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      event = LogStash::Event.new(data)
+      got_event = false
+      subject.on_event do |d|
+        insist { d.chomp } == LogStash::Event.new(data).to_hash.to_edn
+        insist { EDN.read(d)["foo"] } == data["foo"]
+        insist { EDN.read(d)["baz"] } == data["baz"]
+        insist { EDN.read(d)["bah"] } == data["bah"]
+        got_event = true
+      end
+      subject.encode(event)
+      insist { got_event }
+    end
+  end
+
+end

data/spec/codecs/edn_lines.rb

@@ -0,0 +1,53 @@
+require "logstash/codecs/edn_lines"
+require "logstash/event"
+require "insist"
+require "edn"
+
+describe LogStash::Codecs::EDNLines do
+  subject do
+    next LogStash::Codecs::EDNLines.new
+  end
+
+  context "#decode" do
+    it "should return an event from edn data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a", "b", "c"]}}
+      subject.decode(data.to_edn + "\n") do |event|
+        insist { event }.is_a?(LogStash::Event)
+        insist { event["foo"] } == data["foo"]
+        insist { event["baz"] } == data["baz"]
+        insist { event["bah"] } == data["bah"]
+      end
+    end
+
+    it "should return an event from edn data when a newline is recieved" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      subject.decode(data.to_edn) do |event|
+        insist {false}
+      end
+      subject.decode("\n") do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event["foo"] } == data["foo"]
+        insist { event["baz"] } == data["baz"]
+        insist { event["bah"] } == data["bah"]
+      end
+    end
+  end
+
+  context "#encode" do
+    it "should return edn data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      event = LogStash::Event.new(data)
+      got_event = false
+      subject.on_event do |d|
+        insist { d.chomp } == LogStash::Event.new(data).to_hash.to_edn
+        insist { EDN.read(d)["foo"] } == data["foo"]
+        insist { EDN.read(d)["baz"] } == data["baz"]
+        insist { EDN.read(d)["bah"] } == data["bah"]
+        got_event = true
+      end
+      subject.encode(event)
+      insist { got_event }
+    end
+  end
+
+end

data/spec/codecs/graphite.rb

@@ -0,0 +1,96 @@
+require "logstash/codecs/graphite"
+require "logstash/event"
+require "insist"
+
+describe LogStash::Codecs::Graphite do
+  subject do
+    next LogStash::Codecs::Graphite.new
+  end
+
+  context "#decode" do
+    it "should return an event from single full graphite line" do
+      name = Random.srand.to_s(36)
+      value = Random.rand*1000
+      timestamp = Time.now.gmtime.to_i
+      subject.decode("#{name} #{value} #{timestamp}\n") do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event[name] } == value
+      end
+    end
+    
+    it "should return multiple events given multiple graphite formated lines" do
+      total_count = Random.rand(20)
+      names = Array.new(total_count) { Random.srand.to_s(36) }
+      values = Array.new(total_count) { Random.rand*1000 }
+      timestamps = Array.new(total_count) { Time.now.gmtime.to_i }
+      data = Array.new(total_count) {|i| "#{names[i]} #{values[i]} #{timestamps[i]}\n"}
+      counter = 0
+      subject.decode(data.join('')) do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event[names[counter]] } == values[counter]
+        counter = counter+1
+      end
+      insist { counter } == total_count
+    end
+    
+    it "should not return an event until newline is hit" do
+      name = Random.srand.to_s(36)
+      value = Random.rand*1000
+      timestamp = Time.now.gmtime.to_i
+      event_returned = false
+      subject.decode("#{name} #{value} #{timestamp}") do |event|
+        event_returned = true
+      end
+      insist { !event_returned }
+      subject.decode("\n") do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event[name] } == value
+        event_returned = true
+      end
+      insist { event_returned }
+    end
+  end
+  
+  context "#encode" do
+    it "should emit an graphite formatted line" do
+      name = Random.srand.to_s(36)
+      value = Random.rand*1000
+      timestamp = Time.now.gmtime
+      subject.metrics = {name => value}
+      subject.on_event do |event|
+        insist { event.is_a? String }
+        insist { event } == "#{name} #{value} #{timestamp.to_i}\n"
+      end
+      subject.encode(LogStash::Event.new("@timestamp" => timestamp))
+    end
+    
+    it "should treat fields as metrics if fields as metrics flag is set" do
+      name = Random.srand.to_s(36)
+      value = Random.rand*1000
+      timestamp = Time.now.gmtime.to_i
+      subject.fields_are_metrics = true
+      subject.on_event do |event|
+        insist { event.is_a? String }
+        insist { event } == "#{name} #{value} #{timestamp.to_i}\n"
+      end
+      subject.encode(LogStash::Event.new({name => value, "@timestamp" => timestamp}))
+      
+      #even if metrics param is set
+      subject.metrics = {"foo" => 4}
+      subject.encode(LogStash::Event.new({name => value, "@timestamp" => timestamp}))
+    end
+    
+    it "should change the metric name format when metrics_format is set" do
+      name = Random.srand.to_s(36)
+      value = Random.rand*1000
+      timestamp = Time.now.gmtime
+      subject.metrics = {name => value}
+      subject.metrics_format = "foo.bar.*.baz"
+      subject.on_event do |event|
+        insist { event.is_a? String }
+        insist { event } == "foo.bar.#{name}.baz #{value} #{timestamp.to_i}\n"
+      end
+      subject.encode(LogStash::Event.new("@timestamp" => timestamp))
+    end
+  end
+end

data/spec/codecs/json.rb

@@ -0,0 +1,57 @@
+require "logstash/codecs/json"
+require "logstash/event"
+require "insist"
+
+describe LogStash::Codecs::JSON do
+  subject do
+    next LogStash::Codecs::JSON.new
+  end
+
+  context "#decode" do
+    it "should return an event from json data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      subject.decode(data.to_json) do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event["foo"] } == data["foo"]
+        insist { event["baz"] } == data["baz"]
+        insist { event["bah"] } == data["bah"]
+      end
+    end
+
+    it "should be fast" do
+      json = '{"message":"Hello world!","@timestamp":"2013-12-21T07:01:25.616Z","@version":"1","host":"Macintosh.local","sequence":1572456}'
+      iterations = 500000
+      count = 0
+
+      # Warmup
+      10000.times { subject.decode(json) { } }
+
+      start = Time.now
+      iterations.times do
+        subject.decode(json) do |event|
+          count += 1
+        end
+      end
+      duration = Time.now - start
+      insist { count } == iterations
+      puts "codecs/json speed: #{iterations/duration}/sec"
+    end
+  end
+
+  context "#encode" do
+    it "should return json data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      event = LogStash::Event.new(data)
+      got_event = false
+      subject.on_event do |d|
+        insist { d.chomp } == LogStash::Event.new(data).to_json
+        insist { JSON.parse(d)["foo"] } == data["foo"]
+        insist { JSON.parse(d)["baz"] } == data["baz"]
+        insist { JSON.parse(d)["bah"] } == data["bah"]
+        got_event = true
+      end
+      subject.encode(event)
+      insist { got_event }
+    end
+  end
+end

data/spec/codecs/json_lines.rb

@@ -0,0 +1,51 @@
+require "logstash/codecs/json_lines"
+require "logstash/event"
+require "insist"
+
+describe LogStash::Codecs::JSONLines do
+  subject do
+    next LogStash::Codecs::JSONLines.new
+  end
+
+  context "#decode" do
+    it "should return an event from json data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      subject.decode(data.to_json+"\n") do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event["foo"] } == data["foo"]
+        insist { event["baz"] } == data["baz"]
+        insist { event["bah"] } == data["bah"]
+      end
+    end
+    
+    it "should return an event from json data when a newline is recieved" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      subject.decode(data.to_json) do |event|
+        insist {false}
+      end
+      subject.decode("\n") do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event["foo"] } == data["foo"]
+        insist { event["baz"] } == data["baz"]
+        insist { event["bah"] } == data["bah"]
+      end
+    end
+  end
+
+  context "#encode" do
+    it "should return json data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      event = LogStash::Event.new(data)
+      got_event = false
+      subject.on_event do |d|
+        insist { d.chomp } == LogStash::Event.new(data).to_json
+        insist { JSON.parse(d)["foo"] } == data["foo"]
+        insist { JSON.parse(d)["baz"] } == data["baz"]
+        insist { JSON.parse(d)["bah"] } == data["bah"]
+        got_event = true
+      end
+      subject.encode(event)
+      insist { got_event }
+    end
+  end
+end

data/spec/codecs/json_spooler.rb

@@ -0,0 +1,43 @@
+require "logstash/codecs/json_spooler"
+require "logstash/event"
+require "insist"
+
+describe LogStash::Codecs::JsonSpooler do
+  subject do
+    next LogStash::Codecs::JsonSpooler.new
+  end
+
+  context "#decode" do
+    it "should return an event from spooled json data" do
+      data = {"a" => 1}
+      events = [LogStash::Event.new(data), LogStash::Event.new(data),
+        LogStash::Event.new(data)]
+      subject.decode(events.to_json) do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event["a"] } == data["a"]
+      end
+    end
+  end
+
+  context "#encode" do
+    it "should return spooled json data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      subject.spool_size = 3
+      got_event = false
+      subject.on_event do |d|
+        events = JSON.parse(d)
+        insist { events.is_a? Array }
+        insist { events[0].is_a? LogStash::Event }
+        insist { events[0]["foo"] } == data["foo"]
+        insist { events[0]["baz"] } == data["baz"]
+        insist { events[0]["bah"] } == data["bah"]
+        insist { events.length } == 3
+        got_event = true
+      end
+      3.times do
+        subject.encode(LogStash::Event.new(data))
+      end
+      insist { got_event }
+    end
+  end
+end

data/spec/codecs/msgpack.rb

@@ -0,0 +1,39 @@
+require "logstash/codecs/msgpack"
+require "logstash/event"
+require "insist"
+
+# Skip msgpack for now since Hash#to_msgpack seems to not be a valid method?
+describe LogStash::Codecs::Msgpack, :if => false  do
+  subject do
+    next LogStash::Codecs::Msgpack.new
+  end
+
+  context "#decode" do
+    it "should return an event from msgpack data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      subject.decode(data.to_msgpack) do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event["foo"] } == data["foo"]
+        insist { event["baz"] } == data["baz"]
+        insist { event["bah"] } == data["bah"]
+      end
+    end
+  end
+
+  context "#encode" do
+    it "should return msgpack data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      event = LogStash::Event.new(data)
+      got_event = false
+      subject.on_event do |d|
+        insist { d } == LogStash::Event.new(data).to_hash.to_msgpack
+        insist { MessagePack.unpack(d)["foo"] } == data["foo"]
+        insist { MessagePack.unpack(d)["baz"] } == data["baz"]
+        insist { MessagePack.unpack(d)["bah"] } == data["bah"]
+        got_event = true
+      end
+      subject.encode(event)
+      insist { got_event }
+    end
+  end
+end

data/spec/codecs/multiline.rb

@@ -0,0 +1,60 @@
+require "logstash/codecs/multiline"
+require "logstash/event"
+require "insist"
+
+describe LogStash::Codecs::Multiline do
+  context "#decode" do
+    it "should be able to handle multiline events with additional lines space-indented" do
+      codec = LogStash::Codecs::Multiline.new("pattern" => "^\\s", "what" => "previous")
+      lines = [ "hello world", "   second line", "another first line" ]
+      events = []
+      lines.each do |line|
+        codec.decode(line) do |event|
+          events << event
+        end
+      end
+      codec.flush { |e| events << e }
+      insist { events.size } == 2
+      insist { events[0]["message"] } == "hello world\n   second line"
+      insist { events[0]["tags"] }.include?("multiline")
+      insist { events[1]["message"] } == "another first line"
+      insist { events[1]["tags"] }.nil?
+    end
+
+    it "should allow custom tag added to multiline events" do
+      codec = LogStash::Codecs::Multiline.new("pattern" => "^\\s", "what" => "previous", "multiline_tag" => "hurray" )
+      lines = [ "hello world", "   second line", "another first line" ]
+      events = []
+      lines.each do |line|
+        codec.decode(line) do |event|
+          events << event
+        end
+      end
+      codec.flush { |e| events << e }
+      insist { events.size } == 2
+      insist { events[0]["tags"] }.include?("hurray")
+      insist { events[1]["tags"] }.nil?
+    end
+
+    it "should allow grok patterns to be used" do
+      codec = LogStash::Codecs::Multiline.new(
+        "pattern" => "^%{NUMBER} %{TIME}",
+        "negate" => true,
+        "what" => "previous"
+      )
+
+      lines = [ "120913 12:04:33 first line", "second line", "third line" ]
+
+      events = []
+      lines.each do |line|
+        codec.decode(line) do |event|
+          events << event
+        end
+      end
+      codec.flush { |e| events << e }
+
+      insist { events.size } == 1
+      insist { events.first["message"] } == lines.join("\n")
+    end
+  end
+end