metriks-addons 2.1.0

data/lib/metriks/signalfx_reporter.rb

@@ -0,0 +1,182 @@
+require 'metriks/time_tracker'
+require 'rest-client'
+require 'logger'
+
+module Metriks
+  class SignalFxReporter
+    attr_accessor :prefix, :source, :data, :hostname, :tags, :logger
+
+    def initialize(h, token, id, tags, options = {})
+      @hostname = h
+      @x_sf_token = token
+      @orgid = id
+      @tags = tags
+
+      @prefix = options[:prefix]
+      @source = options[:source]
+
+      @logger       = options[:logger] || nil
+      @batch_size   = options[:batch_size] || 50
+      @registry     = options[:registry] || Metriks::Registry.default
+      @interval     = options[:interval] || 60
+      @time_tracker = Metriks::TimeTracker.new(@interval)
+      @on_error     = options[:on_error] || proc { |ex| }
+
+      if options[:percentiles]
+        @percentiles = options[:percentiles]
+      else
+        @percentiles = [ 0.95, 0.99]
+      end
+
+      @mutex = Mutex.new
+      @running = false
+    end
+
+    def log(level, msg)
+      if !@logger.nil?
+        @logger.send level, msg
+      end
+    end
+
+    def start
+      if @thread && @thread.alive?
+        return
+      end
+
+      @running = true
+      @thread = Thread.new do
+        while @running
+          @time_tracker.sleep
+
+          Thread.new do
+            flush
+          end
+        end
+      end
+    end
+
+    def stop
+      @running = false
+
+      if @thread
+        @thread.join
+        @thread = nil
+      end
+    end
+
+    def restart
+      stop
+      start
+    end
+
+    def flush
+      begin
+        @mutex.synchronize do
+          log "debug", "Flushing metrics"
+          submit get_datapoints
+        end
+      rescue Exception => ex
+        log "error",ex.message
+        @on_error[ex] rescue nil
+      end
+    end
+
+    def submit(datapoints)
+      return if datapoints.empty?
+
+      jsonstr = datapoints.to_json
+      response  = RestClient.post "#{@hostname}?orgid=#{@orgid}",
+						        jsonstr,
+						        :content_type => :json, :accept => :json, :'X-SF-TOKEN' => @x_sf_token
+      log "info", "Sent #{datapoints.size} metrics"
+    end
+
+    def get_datapoints
+      time = @time_tracker.now_floored
+
+      datapoints = {}
+      counter = []
+      gauge = []
+      @registry.each do |name, metric|
+        next if name.nil? || name.empty?
+        name = name.to_s.gsub(/ +/, '_')
+        if @prefix
+          name = "#{@prefix}.#{name}"
+        end
+
+        case metric
+        when Metriks::Meter
+          counter |= create_datapoints name, metric, time, [
+            :count, :one_minute_rate, :five_minute_rate,
+            :fifteen_minute_rate, :mean_rate
+          ]
+        when Metriks::Counter
+          counter |= create_datapoints name, metric, time, [
+            :count
+          ]
+        when Metriks::Gauge
+          gauge |= create_datapoints name, metric, time, [
+            :value
+          ]
+        when Metriks::UtilizationTimer
+          counter |= create_datapoints name, metric, time, [
+            :count, :one_minute_rate, :five_minute_rate,
+            :fifteen_minute_rate, :mean_rate,
+            :min, :max, :mean, :stddev,
+            :one_minute_utilization, :five_minute_utilization,
+            :fifteen_minute_utilization, :mean_utilization,
+          ], [
+            :median, :get_95th_percentile
+          ]
+
+          when Metriks::Timer
+          counter |= create_datapoints name, metric, time, [
+            :count, :one_minute_rate, :five_minute_rate,
+            :fifteen_minute_rate, :mean_rate,
+            :min, :max, :mean, :stddev
+          ], [
+            :median, :get_95th_percentile
+          ]
+          when Metriks::Histogram
+          counter |= create_datapoints name, metric, time, [
+            :count, :min, :max, :mean, :stddev
+          ], [
+            :median, :get_95th_percentile
+          ]
+        end
+      end
+      
+      datapoints[:counter] = counter if counter.any?
+      datapoints[:gauge] = gauge if gauge.any?
+
+      datapoints
+    end
+
+    def create_datapoints(base_name, metric, time, keys, snapshot_keys = [])
+      datapoints = []
+      keys.flatten.each do |key|
+        name = key.to_s.gsub(/^get_/, '')
+        datapoints << {
+          :metric => "#{base_name}.#{name}",
+          :timestamp => time,
+          :value => metric.send(key),
+          :dimensions => @tags
+        }
+      end
+
+      unless snapshot_keys.empty?
+        snapshot = metric.snapshot
+        snapshot_keys.flatten.each do |key|
+          name = key.to_s.gsub(/^get_/, '')
+          datapoints << {
+            :metric => "#{base_name}.#{name}",
+            :timestamp => time,
+            :value => snapshot.send(key),
+            :dimensions => @tags
+          }
+        end
+      end
+      datapoints
+    end
+  end
+end

data/metriks-addons.gemspec

@@ -0,0 +1,77 @@
+## This is the rakegem gemspec template. Make sure you read and understand
+## all of the comments. Some sections require modification, and others can
+## be deleted if you don't need them. Once you understand the contents of
+## this file, feel free to delete any comments that begin with two hash marks.
+## You can find comprehensive Gem::Specification documentation, at
+## http://docs.rubygems.org/read/chapter/20
+Gem::Specification.new do |s|
+  s.specification_version = 2 if s.respond_to? :specification_version=
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.rubygems_version = '1.3.5'
+
+  ## Leave these as is they will be modified for you by the rake gemspec task.
+  ## If your rubyforge_project name is different, then edit it and comment out
+  ## the sub! line in the Rakefile
+  s.name              = 'metriks-addons'
+  s.version           = '2.1.0'
+  s.date              = '2015-08-18'
+
+  ## Make sure your summary is short. The description may be as long
+  ## as you like.
+  s.summary     = "Reporters for Metriks."
+  s.description = "OpenTSDB and SignalFX reporter for Metriks."
+
+  ## List the primary authors. If there are a bunch of authors, it's probably
+  ## better to set the email to an email list or something. If you don't have
+  ## a custom homepage, consider using your GitHub URL or the like.
+  s.authors  = ["Rajat Venkatesh"]
+  s.email    = 'rvenkatesh@qubole.com'
+  s.homepage = 'https://github.com/vrajat/metriks-addons'
+
+  ## This gets added to the $LOAD_PATH so that 'lib/NAME.rb' can be required as
+  ## require 'NAME.rb' or'/lib/NAME/file.rb' can be as require 'NAME/file.rb'
+  s.require_paths = %w[lib]
+
+  ## This sections is only necessary if you have C extensions.
+  # s.require_paths << 'ext'
+  # s.extensions = %w[ext/extconf.rb]
+
+  ## If your gem includes any executables, list them here.
+  # s.executables = ["name"]
+
+  ## Specify any RDoc options here. You'll want to add your README and
+  ## LICENSE files to the extra_rdoc_files list.
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.extra_rdoc_files = %w[README.md LICENSE]
+
+  ## List your runtime dependencies here. Runtime dependencies are those
+  ## that are needed for an end user to actually USE your code.
+  s.add_runtime_dependency 'metriks', '~> 0.9', '>= 0.9.9.7'
+  s.add_runtime_dependency 'rest-client', '>= 1.6.7'
+
+  ## List your development dependencies here. Development dependencies are
+  ## those that are only needed during development
+  s.add_development_dependency('mocha', ['~> 0.10'])
+
+  ## Leave this section as-is. It will be automatically generated from the
+  ## contents of your Git repository via the gemspec task. DO NOT REMOVE
+  ## THE MANIFEST COMMENTS, they are used as delimiters by the task.
+  # = MANIFEST =
+  s.files = %w[
+    Gemfile
+    LICENSE
+    README.md
+    Rakefile
+    lib/metriks-addons.rb
+    lib/metriks/opentsdb_reporter.rb
+    lib/metriks/signalfx_reporter.rb
+    metriks-addons.gemspec
+    spec/opentsdb_spec.rb
+    spec/signalfx_spec.rb
+  ]
+  # = MANIFEST =
+
+  ## Test files will be grabbed from the file list. Make sure the path glob
+  ## matches what you actually use.
+  s.test_files = s.files.select { |path| path =~ /^test\/.*_test\.rb/ }
+end

data/spec/opentsdb_spec.rb

@@ -0,0 +1,238 @@
+require 'webmock/rspec'
+require 'metriks'
+require 'metriks/opentsdb_reporter'
+
+describe "Smoke test" do
+  before(:all) do
+    stub_request(:any, "http://localhost:4242")
+  end
+
+  before(:each) do
+    @registry = Metriks::Registry.new
+    @reporter = Metriks::OpenTSDBReporter.new(
+      'http://localhost:4242',
+      {:env => "test"},
+      { :registry => @registry })
+  end
+
+  after(:each) do
+    @reporter.stop
+    @registry.stop
+  end
+
+  it "meter" do
+    @registry.meter('meter.testing').mark
+    datapoints = @reporter.get_datapoints
+    expect(datapoints.size).to eql(5)
+    expect(datapoints[0][:metric]).to eql("meter.testing.count")
+    expect(datapoints[0][:value]).to eql(1)
+    expect(datapoints[0][:tags]).to include(:env => "test")
+    expect(datapoints[0][:timestamp]).not_to be_nil
+
+    expect(datapoints[1][:metric]).to eql("meter.testing.one_minute_rate")
+    expect(datapoints[1][:value]).not_to be_nil
+    expect(datapoints[1][:tags]).to include(:env => "test")
+    expect(datapoints[1][:timestamp]).not_to be_nil
+
+    expect(datapoints[2][:metric]).to eql("meter.testing.five_minute_rate")
+    expect(datapoints[2][:value]).to eql(0.0)
+    expect(datapoints[2][:tags]).to include(:env => "test")
+    expect(datapoints[2][:timestamp]).not_to be_nil
+
+    expect(datapoints[3][:metric]).to eql("meter.testing.fifteen_minute_rate")
+    expect(datapoints[3][:value]).to eql(0.0)
+    expect(datapoints[3][:tags]).to include(:env => "test")
+    expect(datapoints[3][:timestamp]).not_to be_nil
+
+    expect(datapoints[4][:metric]).to eql("meter.testing.mean_rate")
+    expect(datapoints[4][:value]).not_to be_nil
+    expect(datapoints[4][:tags]).to include(:env => "test")
+    expect(datapoints[4][:timestamp]).not_to be_nil
+  end
+
+  it "counter" do
+    @registry.counter('counter.testing').increment
+    datapoints = @reporter.get_datapoints
+    expect(datapoints.size).to eql(1)
+    expect(datapoints[0][:metric]).to eql("counter.testing.count")
+    expect(datapoints[0][:value]).to eql(1)
+    expect(datapoints[0][:tags]).to include(:env => "test")
+    expect(datapoints[0][:timestamp]).not_to be_nil
+  end
+
+  it "timer" do
+    @registry.timer('timer.testing').update(1.5)
+    datapoints = @reporter.get_datapoints
+    expect(datapoints.size).to eql(11)
+    expect(datapoints[0][:metric]).to eql("timer.testing.count")
+    expect(datapoints[0][:value]).to eql(1)
+    expect(datapoints[0][:tags]).to include(:env => "test")
+    expect(datapoints[0][:timestamp]).not_to be_nil
+
+    expect(datapoints[1][:metric]).to eql("timer.testing.one_minute_rate")
+    expect(datapoints[1][:value]).not_to be_nil
+    expect(datapoints[1][:tags]).to include(:env => "test")
+    expect(datapoints[1][:timestamp]).not_to be_nil
+
+    expect(datapoints[2][:metric]).to eql("timer.testing.five_minute_rate")
+    expect(datapoints[2][:value]).to eql(0.0)
+    expect(datapoints[2][:tags]).to include(:env => "test")
+    expect(datapoints[2][:timestamp]).not_to be_nil
+
+    expect(datapoints[3][:metric]).to eql("timer.testing.fifteen_minute_rate")
+    expect(datapoints[3][:value]).to eql(0.0)
+    expect(datapoints[3][:tags]).to include(:env => "test")
+    expect(datapoints[3][:timestamp]).not_to be_nil
+
+    expect(datapoints[4][:metric]).to eql("timer.testing.mean_rate")
+    expect(datapoints[4][:value]).not_to be_nil
+    expect(datapoints[4][:tags]).to include(:env => "test")
+    expect(datapoints[4][:timestamp]).not_to be_nil
+
+    expect(datapoints[5][:metric]).to eql("timer.testing.min")
+    expect(datapoints[5][:value]).not_to be_nil
+    expect(datapoints[5][:tags]).to include(:env => "test")
+    expect(datapoints[5][:timestamp]).not_to be_nil
+
+    expect(datapoints[6][:metric]).to eql("timer.testing.max")
+    expect(datapoints[6][:value]).not_to be_nil
+    expect(datapoints[6][:tags]).to include(:env => "test")
+    expect(datapoints[6][:timestamp]).not_to be_nil
+
+    expect(datapoints[7][:metric]).to eql("timer.testing.mean")
+    expect(datapoints[7][:value]).not_to be_nil
+    expect(datapoints[7][:tags]).to include(:env => "test")
+    expect(datapoints[7][:timestamp]).not_to be_nil
+
+    expect(datapoints[8][:metric]).to eql("timer.testing.stddev")
+    expect(datapoints[8][:value]).not_to be_nil
+    expect(datapoints[8][:tags]).to include(:env => "test")
+    expect(datapoints[8][:timestamp]).not_to be_nil
+
+    expect(datapoints[9][:metric]).to eql("timer.testing.median")
+    expect(datapoints[9][:value]).not_to be_nil
+    expect(datapoints[9][:tags]).to include(:env => "test")
+    expect(datapoints[9][:timestamp]).not_to be_nil
+
+    expect(datapoints[10][:metric]).to eql("timer.testing.95th_percentile")
+    expect(datapoints[10][:value]).not_to be_nil
+    expect(datapoints[10][:tags]).to include(:env => "test")
+    expect(datapoints[10][:timestamp]).not_to be_nil
+  end
+
+  it "histogram" do
+    @registry.histogram('histogram.testing').update(1.5)
+    datapoints = @reporter.get_datapoints
+    expect(datapoints.size).to eql(7)
+    expect(datapoints[0][:metric]).to eql("histogram.testing.count")
+    expect(datapoints[0][:value]).to eql(1)
+    expect(datapoints[0][:tags]).to include(:env => "test")
+    expect(datapoints[0][:timestamp]).not_to be_nil
+
+    expect(datapoints[1][:metric]).to eql("histogram.testing.min")
+    expect(datapoints[1][:value]).not_to be_nil
+    expect(datapoints[1][:tags]).to include(:env => "test")
+    expect(datapoints[1][:timestamp]).not_to be_nil
+
+    expect(datapoints[2][:metric]).to eql("histogram.testing.max")
+    expect(datapoints[2][:value]).not_to be_nil
+    expect(datapoints[2][:tags]).to include(:env => "test")
+    expect(datapoints[2][:timestamp]).not_to be_nil
+
+    expect(datapoints[3][:metric]).to eql("histogram.testing.mean")
+    expect(datapoints[3][:value]).not_to be_nil
+    expect(datapoints[3][:tags]).to include(:env => "test")
+    expect(datapoints[3][:timestamp]).not_to be_nil
+
+    expect(datapoints[4][:metric]).to eql("histogram.testing.stddev")
+    expect(datapoints[4][:value]).not_to be_nil
+    expect(datapoints[4][:tags]).to include(:env => "test")
+    expect(datapoints[4][:timestamp]).not_to be_nil
+
+    expect(datapoints[5][:metric]).to eql("histogram.testing.median")
+    expect(datapoints[5][:value]).not_to be_nil
+    expect(datapoints[5][:tags]).to include(:env => "test")
+    expect(datapoints[5][:timestamp]).not_to be_nil
+
+    expect(datapoints[6][:metric]).to eql("histogram.testing.95th_percentile")
+    expect(datapoints[6][:value]).not_to be_nil
+    expect(datapoints[6][:tags]).to include(:env => "test")
+    expect(datapoints[6][:timestamp]).not_to be_nil
+  end
+
+  it "gauge" do
+    @registry.gauge('gauge.testing') { 123 }
+    datapoints = @reporter.get_datapoints
+    expect(datapoints.size).to eql(1)
+    expect(datapoints[0][:metric]).to eql("gauge.testing.value")
+    expect(datapoints[0][:value]).to eql(123)
+    expect(datapoints[0][:tags]).to include(:env => "test")
+    expect(datapoints[0][:timestamp]).not_to be_nil
+  end
+end
+
+describe "Rest Client" do
+  before(:each) do
+    @registry = Metriks::Registry.new
+    @reporter = Metriks::OpenTSDBReporter.new(
+      'http://localhost:4242',
+      {:env => "test"},
+      { :registry => @registry, :batch_size => 3})
+    stub_request(:post, "http://localhost:4242/api/put").
+      with(:body => /^\[.*\]$/).
+      to_return(:status => 200, :body => "", :headers => {})
+  end
+
+  it "Send a single metric" do
+    @registry.counter('counter.testing').increment
+    @reporter.submit @reporter.get_datapoints
+    #TODO Also check body
+    body = [{
+      :metric => "counter.testing",
+      :timestamp => 1438446900,
+      :value => 1,
+      :tags => { :env => "test"}}
+    ]
+    expect(a_request(:post, "http://localhost:4242/api/put")).to have_been_made
+  end
+
+  it "Send a three metric" do
+    for i in 0..2 do
+      @registry.counter("counter.testing.#{i}").increment
+    end
+    @reporter.submit @reporter.get_datapoints
+    expect(a_request(:post, "http://localhost:4242/api/put")).to have_been_made
+  end
+
+  it "Send a four metric" do
+    for i in 0..3 do
+      @registry.counter("counter.testing.#{i}").increment
+    end
+    @reporter.submit @reporter.get_datapoints
+    expect(a_request(:post, "http://localhost:4242/api/put")).to have_been_made.times(2)
+  end
+
+  it "Send a five metric" do
+    for i in 0..4 do
+      @registry.counter("counter.testing.#{i}").increment
+    end
+    @reporter.submit @reporter.get_datapoints
+    expect(a_request(:post, "http://localhost:4242/api/put")).to have_been_made.times(2)
+  end
+
+  it "Send a six metric" do
+    for i in 0..5 do
+      @registry.counter("counter.testing.#{i}").increment
+    end
+    @reporter.submit @reporter.get_datapoints
+    expect(a_request(:post, "http://localhost:4242/api/put")).to have_been_made.times(2)
+  end
+
+  it "Send a seven metric" do
+    for i in 0..7 do
+      @registry.counter("counter.testing.#{i}").increment
+    end
+    @reporter.submit @reporter.get_datapoints
+    expect(a_request(:post, "http://localhost:4242/api/put")).to have_been_made.times(3)
+  end
+end