fluent-plugin-librato-metrics 0.2.2

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/ChangeLog

@@ -0,0 +1,5 @@
+
+Release 0.2.2 - 2012/03/29
+
+* First release
+

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in fluent-plugin-librato-metrics.gemspec
+gemspec

data/README.rdoc

@@ -0,0 +1,51 @@
+= Librato Metrics output plugin for Fluent event collector
+
+== Overview
+
+*librato_metrics* output plugin buffers aggregate logs and upload them to Librato Metrics periodically.
+
+== Installation
+
+Simply use RubyGems:
+
+    gem install fluent-plugin-librato-metrics
+
+== Configuration
+
+    <match pattern>
+      type librato_metrics
+
+      librato_user YOUR_LIBRATO_USER_NAME
+      librato_token YOUR_LIBRATO_TOKEN
+
+      # simple key-value metrics
+      <metrics metrics.status.**>
+        name status
+        each_key key
+        value_key value
+      </metrics>
+
+      # single-stream event count
+      <metrics metrics.event.mytask>
+        name mytask.count
+      </metrics>
+
+      # single-stream event sum
+      <metrics metrics.event.mytask>
+        name mytask.size
+        value_key size
+      </metrics>
+
+      # single-stream event sum; value type is float
+      <metrics metrics.event.mytask>
+        name mytask.elapsed
+        value_key elapsed
+        type float
+      </metrics>
+
+    </match>
+
+[librato_user (required)] User name of librato metrics
+
+[librato_token (required)] Token of librato metrics
+

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.2.2

data/fluent-plugin-librato-metrics.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "fluent-plugin-librato-metrics"
+  s.version     = File.read("VERSION").strip
+  s.authors     = ["Sadayuki Furuhashi"]
+  s.email       = ["frsyuki@gmail.com"]
+  #s.homepage    = "https://github.com/fluent/fluent-plugin-librato-metrics"  # TODO
+  s.summary     = %q{Librato metrics output plugin for Fluent event collector}
+  s.description = %q{Librato metrics output plugin for Fluent event collector}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+  s.add_dependency "fluentd", "~> 0.10.0"
+  s.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/out_librato_metrics.rb

@@ -0,0 +1,148 @@
+#
+# fluent-plugin-librato-metrics
+#
+# Copyright (C) 2011-2012 Sadayuki Furuhashi
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class LibratoMetricsOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('librato_metrics', self)
+
+  require 'net/http'
+  require "#{File.dirname(__FILE__)}/out_librato_metrics_mixin"
+  include MetricsMixin
+
+  config_param :librato_user, :string
+  config_param :librato_token, :string
+  config_param :source, :string, :default => nil
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def format(tag, time, record)
+    out = ''.force_encoding('ASCII-8BIT')
+    each_metrics(tag, time, record) {|d|
+      name = d.name
+      if d.each_keys.empty?
+        key = name
+      else
+        key = "#{name}.#{d.keys.join('.')}"
+      end
+      partitioned_time = time / 60 * 60
+      [key, partitioned_time, d.value*d.count].to_msgpack(out)
+    }
+    out
+  end
+
+  def write(chunk)
+    counters = {}  #=> {partitioned_time => {name => CounterAggregator}}
+    gauges = {}    #=> {partitioned_time => {name => GaugeAggregator}}
+
+    chunk.msgpack_each {|key,partitioned_time,value|
+      if m = /^counter\.(.*)$/.match(key)
+        name = m[1]
+        ((counters[partitioned_time] ||= {})[name] ||= CounterAggregator.new).add(value)
+      elsif m = /^gauge\.(.*)$/.match(key)
+        name = m[1]
+        ((gauges[partitioned_time] ||= {})[name] ||= GaugeAggregator.new).add(value)
+      else
+        name = key
+        ((gauges[partitioned_time] ||= {})[name] ||= GaugeAggregator.new).add(value)
+      end
+    }
+
+    #http = Net::HTTP.new('metrics-api.librato.com', 80)
+    http = Net::HTTP.new('metrics-api.librato.com', 443)
+    http.use_ssl = true
+    #http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE  # TODO verify
+    http.cert_store = OpenSSL::X509::Store.new
+    header = {}
+
+    begin
+      {'counters'=>counters, 'gauges'=>gauges}.each_pair {|type,partitions|
+        partitions.each_pair {|partitioned_time,name_aggrs|
+          req = Net::HTTP::Post.new('/v1/metrics', header)
+          req.basic_auth @librato_user, @librato_token
+
+          params = {
+            'measure_time' => partitioned_time.to_s,
+          }
+          params['source'] = @source if @source
+
+          name_aggrs.each_with_index {|(name,aggr),i|
+            params["#{type}[#{i}][name]"] = name
+            params["#{type}[#{i}][value]"] = aggr.get.to_s
+          }
+
+          $log.trace { "librato metrics: #{params.inspect}" }
+          req.set_form_data(params)
+          res = http.request(req)
+
+          # TODO error handling
+          if res.code != "200"
+            $log.warn "librato_metrics: #{res.code}: #{res.body}"
+          end
+        }
+      }
+    ensure
+      http.finish if http.started?
+    end
+  end
+
+
+  # max(value) aggregator
+  class CounterAggregator
+    def initialize
+      @value = 0
+    end
+
+    def add(value)
+      @value = value if @value < value
+    end
+
+    def get
+      # librato metrics's counter only supports integer:
+      # 'invalid value for Integer(): "0.06268015"'
+      @value.to_i
+    end
+  end
+
+  # avg(value) aggregator
+  class GaugeAggregator
+    def initialize
+      @value = 0
+      @num = 0
+    end
+
+    def add(value)
+      @value += value
+      @num += 1
+    end
+
+    def get
+      @value / @num
+    end
+  end
+end
+
+
+end

data/lib/fluent/plugin/out_librato_metrics_mixin.rb

@@ -0,0 +1,317 @@
+#
+# fluent-plugin-librato-metrics
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+module LibratoMetricsOutput::MetricsMixin
+  include Configurable
+
+  def initialize
+    super
+    @metrics = []
+  end
+
+  attr_reader :metrics
+
+  config_param :default_value_key, :string, :default => nil
+  config_param :default_count_key, :string, :default => nil
+  config_param :default_type, :string, :default => nil
+  config_param :match_key, :string, :default => nil
+
+  def configure(conf)
+    super
+
+    defaults = {
+      'value_key' => @default_value_key,
+      'count_key' => @default_count_key,
+      'type' => @default_type,
+    }
+
+    conf.elements.select {|e|
+      e.name == 'metrics'
+    }.each {|e|
+      defaults.each_pair {|k,v| e[k] = v if v != nil }
+      add_metrics(e, e.arg)
+    }
+
+    if mk = @match_key
+      @match_key_proc = Proc.new {|tag,record| record[mk] }
+    else
+      @match_key_proc = Proc.new {|tag,record| tag }
+    end
+  end
+
+  def add_metrics(conf, pattern)
+    m = Metrics.new(pattern)
+    m.configure(conf)
+    @metrics << m
+  end
+
+  class Data
+    def initialize(metrics)
+      @metrics = metrics
+      @name = metrics.name
+      @each_keys = metrics.each_keys
+      @value_key = metrics.value_key
+      @count_key = metrics.count_key
+    end
+
+    attr_reader :metrics
+    attr_reader :name, :each_keys, :value_key, :count_key
+    attr_accessor :keys, :value, :count
+  end
+
+  class Metrics
+    include Configurable
+
+    def initialize(pattern)
+      super()
+      if pattern && !pattern.empty?
+        @pattern = MatchPattern.create(pattern)
+      else
+        @pattern = nil
+      end
+    end
+
+    attr_reader :pattern
+
+    config_param :name, :string
+
+    config_param :each_key, :string, :default => nil
+    attr_reader :each_keys  # parsed 'each_key'
+
+    config_param :value_key, :string, :default => nil
+    config_param :default_value, :string, :default => nil
+
+    config_param :count_key, :string, :default => nil
+
+    config_param :type, :default => :int do |val|
+      case val
+      when 'int'
+        :int
+      when 'float'
+        :float
+      else
+        raise ConfigError, "unexpected 'type' parameter on metrics: expected int or float"
+      end
+    end
+
+    config_param :match_key, :string, :default => nil
+
+    attr_accessor :key_proc
+    attr_accessor :value_proc
+    attr_accessor :count_proc
+    attr_accessor :match_proc
+
+    def configure(conf)
+      super
+
+      if @each_key
+        @each_keys = @each_key.split(',').map {|e| e.strip }
+        @key_proc = create_combined_key_proc(@each_keys)
+      else
+        @each_keys = []
+        use_keys = [@name]
+        @key_proc = Proc.new {|record| use_keys }
+      end
+
+      case @type
+      when :int
+        if vk = @value_key
+          dv = @default_value ? @default_value.to_i : nil
+          @value_proc = Proc.new {|record| val = record[vk]; val ? val.to_i : dv }
+        else
+          dv = @default_value ? @default_value.to_i : 1
+          @value_proc = Proc.new {|record| dv }
+        end
+      when :float
+        if vk = @value_key
+          dv = @default_value ? @default_value.to_f : nil
+          @value_proc = Proc.new {|record| val = record[vk]; val ? val.to_f : dv }
+        else
+          dv = @default_value ? @default_value.to_f : 1.0
+          @value_proc = Proc.new {|record| dv }
+        end
+      end
+
+      if ck = @count_key
+        @count_proc = Proc.new {|record| val = record[ck].to_i; val == 0 ? 1 : val }
+      else
+        @count_proc = Proc.new {|record| 1 }
+      end
+
+      if pt = @pattern
+        @match_proc = Proc.new {|tag| pt.match(tag) }
+      else
+        @match_proc = Proc.new {|tag| true }
+      end
+
+      @data = Data.new(self)
+    end
+
+    def create_combined_key_proc(keys)
+      if keys.length == 1
+        key = keys[0]
+        Proc.new {|record|
+          val = record[key]
+          val ? [val] : nil
+        }
+      else
+        Proc.new {|record|
+          keys.map {|key|
+            val = record[key]
+            break nil unless val
+            val
+          }
+        }
+      end
+    end
+
+    def evaluate(tag, time, record)
+      return nil unless @match_proc.call(tag)
+
+      data = @data
+
+      keys = data.keys = @key_proc.call(record)
+      return nil unless keys
+
+      value = data.value = @value_proc.call(record)
+      return nil unless value
+
+      data.count = @count_proc.call(record)
+
+      return data
+    end
+  end
+
+  def each_metrics(tag, time, record, &block)
+    mk = @match_key_proc.call(tag, record)
+    return [] unless mk
+    @metrics.map {|m|
+      data = m.evaluate(mk, time, record)
+      next unless data
+      block.call(data)
+    }
+  end
+end
+
+
+## Example
+#
+#class MetricsOutput < Output
+#  Plugin.register_output('metrics', self)
+#
+#  include MetricsMixin
+#
+#  def initialize
+#    super
+#  end
+#
+#  config_param :tag, :string, :default => nil
+#  config_param :tag_prefix, :string, :default => nil
+#
+#  def configure(conf)
+#    super
+#
+#    if constant_tag = @tag
+#      @tag_proc = Proc.new {|name,tag| constant_tag }
+#    elsif tag_prefix = @tag_prefix
+#      @tag_proc = Proc.new {|name,tag| "#{tag_prefix}.#{name}.#{tag}" }
+#    else
+#      @tag_proc = Proc.new {|name,tag| "#{name}.#{tag}" }
+#    end
+#  end
+#
+#  def emit(tag, es, chain)
+#    es.each {|time,record|
+#      each_metrics(tag, time, record) {|m|
+#        otag = @tag_proc.call(m.name, tag)
+#        orecord = {
+#          'name' => m.name,
+#          'key' => m.keys.join(','),
+#          'keys' => m.keys,
+#          'key_hash' => Hash[ m.each_keys.zip(m.keys) ],
+#          'value' => m.value,
+#          'value_hash' => {m.value_key => m.value},
+#          'count' => m.count,
+#        }
+#        Engine.emit(otag, time, orecord)
+#      }
+#    }
+#    chain.next
+#  end
+#end
+
+
+## Example
+#
+#require 'metrics'
+#
+#class ExampleAggregationOutput < BufferedOutput
+#  Plugin.register_output('aggregation_example', self)
+#
+#  include MetricsMixin
+#
+#  def initialize
+#    super
+#  end
+#
+#  def configure(conf)
+#    super
+#
+#    # do something ...
+#  end
+#
+#  def format(tag, time, record)
+#    # call each_metrics(tag, time,record):
+#    each_metrics(tag, time, record) {|m|
+#      # 'm.name' is always avaiable
+#      name = m.name
+#
+#      # 'keys' is an array of record value if each_key option is specified.
+#      # otherwise, 'keys' is same as [m.name]
+#      key = m.keys.join(',')
+#
+#      if !m.each_keys.empty?
+#        # you can create Hash of the record using 'm.each_keys'
+#        key_hash = Hash[ m.each_keys.zip(m.keys) ]
+#      else
+#        # m.keys is [name]
+#      end
+#
+#      if m.value_key
+#        # value is parsed record value
+#        value_hash = {m.value_key => m.value}
+#      else
+#        # m.value is 1 or 1.0
+#      end
+#
+#      if m.count_key
+#        # value is parsed record value
+#        count_hash = {m.count_key => m.count}
+#      else
+#        # count is 1
+#      end
+#
+#      # ...
+#    }
+#  end
+#end
+
+
+end

data/test/out_librato_metrics.rb

@@ -0,0 +1,81 @@
+$:.push File.expand_path("../lib", __FILE__)
+require 'fluent/test'
+require 'fluent/plugin/out_librato_metrics'
+
+class LibratoMetricsOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    librato_user test
+    librato_token TEST
+
+    # simple key-value metrics
+    <metrics test1.**>
+      name test1
+      each_key key
+      value_key value
+      count_key count
+    </metrics>
+
+    # single-stream event count
+    <metrics test2.**>
+      name test2.count
+    </metrics>
+
+    # single-stream event sum
+    <metrics test3.**>
+      name test3.size
+      value_key size
+    </metrics>
+
+    # single-stream event sum; value type is float
+    <metrics test3.test4>
+      name test4.elapsed
+      value_key elapsed
+      type float
+    </metrics>
+  ]
+
+  TIME = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+  def create_driver(tag="test", conf=CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::LibratoMetricsOutput, tag) do
+      def write(chunk)
+        chunk.read
+      end
+    end.configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'test', d.instance.librato_user
+    assert_equal 'TEST', d.instance.librato_token
+  end
+
+  def test_format
+    d = create_driver("test1.1")
+    d.emit({"key"=>"a", "value"=>10, "count"=>2}, TIME)
+    d.expect_format ["test1.a", TIME/60*60, 20].to_msgpack
+    d.run
+
+    d = create_driver("test2.2")
+    d.emit({}, TIME)
+    d.expect_format ["test2.count", TIME/60*60, 1].to_msgpack
+    d.run
+
+    d = create_driver("test3.3")
+    d.emit({"size"=>2100}, TIME)
+    d.expect_format ["test3.size", TIME/60*60, 2100].to_msgpack
+    d.run
+
+    d = create_driver("test3.test4")
+    d.emit({"size"=>2400, "elapsed"=>10.0}, TIME)
+    d.expect_format ["test3.size", TIME/60*60, 2400].to_msgpack
+    d.expect_format ["test4.elapsed", TIME/60*60, 10.0].to_msgpack
+    d.run
+  end
+
+end
+

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-librato-metrics
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+  prerelease: 
+platform: ruby
+authors:
+- Sadayuki Furuhashi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-30 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: &70131020756540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70131020756540
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70131020756060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: *70131020756060
+description: Librato metrics output plugin for Fluent event collector
+email:
+- frsyuki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- ChangeLog
+- Gemfile
+- README.rdoc
+- Rakefile
+- VERSION
+- fluent-plugin-librato-metrics.gemspec
+- lib/fluent/plugin/out_librato_metrics.rb
+- lib/fluent/plugin/out_librato_metrics_mixin.rb
+- test/out_librato_metrics.rb
+homepage: 
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -197996435032986197
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -197996435032986197
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.12
+signing_key: 
+specification_version: 3
+summary: Librato metrics output plugin for Fluent event collector
+test_files:
+- test/out_librato_metrics.rb