logstash-output-graphite 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49103b7165665819b05df09701b9863faccedc9b
-  data.tar.gz: 6a814793a21fcc8127803e0995d06dd2672d0871
+  metadata.gz: aaa1548ced5cd4e3cc3f3724c078a13e9c507fe5
+  data.tar.gz: 9c2309ed86b7c371bd35ad4d21ed7bb96536b48d
 SHA512:
-  metadata.gz: 086f17b7bdf39a6de526947d03e23f9db690397b34d92eb7de4bf1610d5b139bfc83a0b61b9ceb339f4e546e891f16d659df50c8bf99cec574f09b92e4a31827
-  data.tar.gz: a49c4df036d303676cdc6e87d841d1a8f6d091b06ede88011578978116a8473e1b9b0fa12504e3f85648d4e9f172267a1f9b350d0c9734aa7a6372102b6ff2d4
+  metadata.gz: 26aedf2dae122c104225a6297484edd858457e553b340113d2458c2c4ec963a80133e8505f52cc0e348135db0ab5eb6ee654e8b15419906a1ba7a3f2f62dc7f4
+  data.tar.gz: dd176d82e9c199bb713dc8c8e3ecdda2b30117692994b9beef372d6531c125a6ff1056ae13d6bec070fc62d33f80b4637f8b567dd9404fef663d843a2f9a7571

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+- 1.0.1
+  * Added support for nested hashes as values

data/lib/logstash/outputs/graphite.rb

@@ -67,6 +67,18 @@ class LogStash::Outputs::Graphite < LogStash::Outputs::Base
   # NOTE: If no metrics_format is defined, the name of the metric will be used as fallback.
   config :metrics_format, :validate => :string, :default => DEFAULT_METRICS_FORMAT
 
+  # When hashes are passed in as values they are broken out into a dotted notation
+  # For instance if you configure this plugin with
+  # # [source,ruby]
+  #     metrics => "mymetrics"
+  #
+  # and "mymetrics" is a nested hash of '{a => 1, b => { c => 2 }}'
+  # this plugin will generate two metrics: a => 1, and b.c => 2 .
+  # If you've specified a 'metrics_format' it will respect that,
+  # but you still may want control over the separator within these nested key names.
+  # This config setting changes the separator from the '.' default.
+  config :nested_object_separator, :validate => :string, :default => "."
+
   def register
     @include_metrics.collect!{|regexp| Regexp.new(regexp)}
     @exclude_metrics.collect!{|regexp| Regexp.new(regexp)}
@@ -106,26 +118,9 @@ class LogStash::Outputs::Graphite < LogStash::Outputs::Base
 
     # Graphite message format: metric value timestamp\n
 
-    messages = []
-    timestamp = event[@timestamp_field].to_i
-
-    if @fields_are_metrics
-      @logger.debug("got metrics event", :metrics => event.to_hash)
-      event.to_hash.each do |metric,value|
-        next if EXCLUDE_ALWAYS.include?(metric)
-        next unless @include_metrics.empty? || @include_metrics.any? { |regexp| metric.match(regexp) }
-        next if @exclude_metrics.any? {|regexp| metric.match(regexp)}
-        messages << "#{construct_metric_name(metric)} #{event.sprintf(value.to_s).to_f} #{timestamp}"
-      end
-    else
-      @metrics.each do |metric, value|
-        @logger.debug("processing", :metric => metric, :value => value)
-        metric = event.sprintf(metric)
-        next unless @include_metrics.any? {|regexp| metric.match(regexp)}
-        next if @exclude_metrics.any? {|regexp| metric.match(regexp)}
-        messages << "#{construct_metric_name(event.sprintf(metric))} #{event.sprintf(value).to_f} #{timestamp}"
-      end
-    end
+    messages = @fields_are_metrics ?
+      messages_from_event_fields(event, @include_metrics, @exclude_metrics) :
+      messages_from_event_metrics(event, @metrics)
 
     if messages.empty?
       @logger.debug("Message is empty, not sending anything to Graphite", :messages => messages, :host => @host, :port => @port)
@@ -145,6 +140,69 @@ class LogStash::Outputs::Graphite < LogStash::Outputs::Base
         retry if @resend_on_failure
       end
     end
-
   end # def receive
+
+  private
+
+  def messages_from_event_fields(event, include_metrics, exclude_metrics)
+    timestamp = event_timestamp(event)
+    @logger.debug? && @logger.debug("got metrics event", :metrics => event.to_hash)
+    event.to_hash.flat_map do |metric,value|
+      next if EXCLUDE_ALWAYS.include?(metric)
+      next unless include_metrics.empty? || include_metrics.any? { |regexp| metric.match(regexp) }
+      next if exclude_metrics.any? {|regexp| metric.match(regexp)}
+
+      metrics_lines_for_event(event, metric, value, timestamp)
+    end
+  end
+
+  def messages_from_event_metrics(event, metrics)
+    timestamp = event_timestamp(event)
+    metrics.flat_map do |metric, value|
+      @logger.debug("processing", :metric => metric, :value => value)
+      metric = event.sprintf(metric)
+      next unless @include_metrics.any? {|regexp| metric.match(regexp)}
+      next if @exclude_metrics.any? {|regexp| metric.match(regexp)}
+
+      metrics_lines_for_event(event, metric, value, timestamp)
+    end
+  end
+
+  def event_timestamp(event)
+    event[@timestamp_field].to_i
+  end
+
+  def metrics_lines_for_event(event, metric, value, timestamp)
+    if event[metric].is_a?(Hash)
+      dotify(event[metric], metric).map do |k,v|
+        metrics_line(k, v, timestamp)
+      end
+    else
+      metrics_line(event.sprintf(metric), event.sprintf(value).to_f, timestamp)
+    end
+  end
+
+  def metrics_line(name, value, timestamp)
+    "#{construct_metric_name(name)} #{value} #{timestamp}"
+  end
+
+  # Take a nested ruby hash of the form {:a => {:b => 2}, c: => 3} and
+  # turn it into a hash of the form
+  # { "a.b" => 2, "c" => 3}
+  def dotify(hash,prefix=nil)
+    hash.reduce({}) do |acc,kv|
+      k,v = kv
+      pk = prefix ? "#{prefix}#{@nested_object_separator}#{k}" : k.to_s
+      if v.is_a?(Hash)
+        acc.merge!(dotify(v, pk))
+      elsif v.is_a?(Array)
+        # There's no right answer here, so we do nothing
+        @logger.warn("Array values not supported for graphite metrics! Ignoring #{hash} @ #{prefix}")
+      else
+        acc[pk] = v
+      end
+      acc
+    end
+  end
+
 end # class LogStash::Outputs::Graphite

data/logstash-output-graphite.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-graphite'
-  s.version         = '1.0.0'
+  s.version         = '1.0.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output allows you to pull metrics from your logs and ship them to Graphite"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/graphite_spec.rb

@@ -36,15 +36,25 @@ describe LogStash::Outputs::Graphite do
       let(:event) { LogStash::Event.new("foo" => "123") }
 
       context "match one key" do
-        it "generate one element" do
+        it "should generate one element" do
           expect(server.size).to eq(1)
         end
 
-        it "match the generated key" do
+        it "should match the generated key" do
           line = server.pop
           expect(line).to match(/^foo.bar.sys.data.foo 123.0 \d{10,}\n$/)
         end
       end
+
+      context "when matching a nested hash" do
+        let(:event) { LogStash::Event.new("foo" => {"a" => 3, "c" => {"d" => 2}}) }
+
+        it "should create the proper formatted lines" do
+          lines = [server.pop, server.pop].sort # Put key 'a' first
+          expect(lines[0]).to match(/^foo.bar.sys.data.foo.a 3 \d{10,}\n$/)
+          expect(lines[1]).to match(/^foo.bar.sys.data.foo.c.d 2 \d{10,}\n$/)
+        end
+      end
     end
 
     context "match all keys" do
@@ -124,6 +134,16 @@ describe LogStash::Outputs::Graphite do
           line = server.pop
           expect(line).to match(/^custom.foo 123.0 \d{10,}\n$/)
         end
+
+        context "when matching a nested hash" do
+          let(:event) { LogStash::Event.new("custom.foo" => {"a" => 3, "c" => {"d" => 2}}) }
+
+          it "should create the proper formatted lines" do
+            lines = [server.pop, server.pop].sort # Put key 'a' first
+            expect(lines[0]).to match(/^custom.foo.a 3 \d{10,}\n$/)
+            expect(lines[1]).to match(/^custom.foo.c.d 2 \d{10,}\n$/)
+          end
+        end
       end
     end
   end
@@ -144,4 +164,34 @@ describe LogStash::Outputs::Graphite do
       expect(line).to match(/^foo 1.0 #{timestamp_new}\n$/)
     end
   end
+
+  describe "dotifying a hash" do
+    let(:event) { LogStash::Event.new( "metrics" => hash) }
+    let(:dotified) { LogStash::Outputs::Graphite.new().send(:dotify, hash) }
+
+    context "with a complex hash" do
+      let(:hash) { {:a => 2, :b => {:c => 3, :d => 4, :e => {:f => 5}}} }
+
+      it "should dottify correctly" do
+        expect(dotified).to eql({"a" => 2, "b.c" => 3, "b.d" => 4, "b.e.f" => 5})
+      end
+    end
+
+    context "with a simple hash" do
+      let(:hash) { {:a => 2, 5 => 4} }
+
+      it "should do nothing more than stringify the keys" do
+        expect(dotified).to eql("a" => 2, "5" => 4)
+      end
+    end
+
+    context "with an array value" do
+      let(:hash) { {:a => 2, 5 => 4, :c => [1,2,3]} }
+
+      it "should ignore array values" do
+        expect(dotified).to eql("a" => 2, "5" => 4)
+      end
+    end
+
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-graphite
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -127,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: This output allows you to pull metrics from your logs and ship them to Graphite