fluent-plugin-growthforecast 0.1.6 → 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ec668da8275088e03eff43e7a2af96f2f36d367f
+  data.tar.gz: 772aa71eec3488cbd39e938da961849345c6e2ae
+SHA512:
+  metadata.gz: 3283857ee36842f64a8af77b75486caf5f93827d452b8d912d0e1f829a4d7517d694f441914a69d8b2faa916bfa5d27d3ecfffab7fd743ecd1fadbacce69ae10
+  data.tar.gz: ce10e80a9991f8aba61a65e56dceb6156ed44c0c399ee08c483f4c31ab8684b0acbaef96bbff35015b7ccda3e28eab6387116654c0b94b9d65894ac1093519f9

data/README.md

@@ -71,6 +71,17 @@ If your GrowthForecast protected with basic authentication, specify `authenticat
       password secret!
     </match>
 
+Version v0.2.0 or later, this plugin uses HTTP connection keep-alive for a batch emitted events. To disable this, specify `keepalive` option:
+
+    <match metrics.**>
+      type growthforecast
+      gfapi_url http://growthforecast.protected.anywhere.example.com/api/
+      service   yourservice
+      tag_for   section
+      name_keys fieldname
+	  keepalive no
+    </match>
+
 ## TODO
 
 * patches welcome!

data/fluent-plugin-growthforecast.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-growthforecast"
-  gem.version       = "0.1.6"
+  gem.version       = "0.2.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd output plugin to post numbers to GrowthForecast (by kazeburo)}

data/lib/fluent/plugin/out_growthforecast.rb

@@ -21,7 +21,9 @@ class Fluent::GrowthForecastOutput < Fluent::Output
 
   config_param :remove_prefix, :string, :default => nil
   config_param :tag_for, :string, :default => 'name_prefix' # or 'ignore' or 'section' or 'service'
-  
+
+  config_param :keepalive, :bool, :default => true
+
   config_param :authentication, :string, :default => nil # nil or 'none' or 'basic'
   config_param :username, :string, :default => ''
   config_param :password, :string, :default => ''
@@ -104,23 +106,42 @@ class Fluent::GrowthForecastOutput < Fluent::Output
     end
   end
 
+  def connect_to(tag, name)
+    url = URI.parse(format_url(tag,name))
+    return url.host, url.port
+  end
+
+  def http_connection(host, port)
+    http = Net::HTTP.new(host, port)
+    if @ssl
+      http.use_ssl = true
+      unless @verify_ssl
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+    end
+    http
+  end
+
+  def post_request(tag, name, value)
+    url = URI.parse(format_url(tag,name))
+    req = Net::HTTP::Post.new(url.path)
+    if @auth and @auth == :basic
+      req.basic_auth(@username, @password)
+    end
+    if @keepalive
+      req['Connection'] = 'Keep-Alive'
+    end
+    req.set_form_data({'number' => value.to_i, 'mode' => @mode.to_s})
+    req
+  end
+
   def post(tag, name, value)
     url = format_url(tag,name)
     res = nil
     begin
-      url = URI.parse(url)
-      req = Net::HTTP::Post.new(url.path)
-      if @auth and @auth == :basic
-        req.basic_auth(@username, @password)
-      end
-      req.set_form_data({'number' => value.to_i, 'mode' => @mode.to_s})
-      http = Net::HTTP.new(url.host, url.port)
-      if @ssl
-        http.use_ssl = true
-        unless @verify_ssl
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        end
-      end
+      host,port = connect_to(tag, name)
+      req = post_request(tag, name, value)
+      http = http_connection(host, port)
       res = http.start {|http| http.request(req) }
     rescue IOError, EOFError, SystemCallError
       # server didn't respond
@@ -131,12 +152,35 @@ class Fluent::GrowthForecastOutput < Fluent::Output
     end
   end
 
+  def post_keepalive(events) # [{:tag=>'',:name=>'',:value=>X}]
+    return if events.size < 1
+
+    # gf host/port is same for all events (host is from configuration)
+    host,port = connect_to(events.first[:tag], events.first[:name])
+
+    requests = events.map{|e| post_request(e[:tag], e[:name], e[:value])}
+    begin
+      http = http_connection(host, port)
+      http.start do |http|
+        requests.each do |req|
+          res = http.request(req)
+          unless res and res.is_a?(Net::HTTPSuccess)
+            $log.warn "failed to post to growthforecast: #{host}:#{port}#{req.path}, post_data: #{req.body} code: #{res && res.code}"
+          end
+        end
+      end
+    rescue IOError, EOFError, SystemCallError
+      $log.warn "Net::HTTP.post_form raises exception: #{$!.class}, '#{$!.message}'"
+    end
+  end
+
   def emit(tag, es, chain)
+    events = []
     if @name_keys
       es.each {|time,record|
         @name_keys.each {|name|
           if record[name]
-            post(tag, name, record[name])
+            events.push({:tag => tag, :name => name, :value => record[name]})
           end
         }
       }
@@ -144,11 +188,19 @@ class Fluent::GrowthForecastOutput < Fluent::Output
       es.each {|time,record|
         record.keys.each {|key|
           if @name_key_pattern.match(key) and record[key]
-            post(tag, key, record[key])
+            events.push({:tag => tag, :name => key, :value => record[key]})
           end
         }
       }
     end
+    if @keepalive
+      post_keepalive(events)
+    else
+      events.each do |event|
+        post(event[:tag], event[:name], event[:value])
+      end
+    end
+
     chain.next
   end
 end

data/test/plugin/test_out_growthforecast.rb

@@ -37,7 +37,7 @@ class GrowthForecastOutputTest < Test::Unit::TestCase
       tag_for   service
       remove_prefix test
   ]
-  
+
   CONFIG_SPACE = %[
       gfapi_url http://127.0.0.1:5125/api/
       service   service x
@@ -46,6 +46,15 @@ class GrowthForecastOutputTest < Test::Unit::TestCase
       tag_for   ignore
   ]
 
+  CONFIG_NON_KEEPALIVE = %[
+      gfapi_url http://127.0.0.1:5125/api/
+      service   service
+      section   metrics
+      name_keys field1,field2,otherfield
+      tag_for   name_prefix
+      keepalive false
+  ]
+
   def create_driver(conf=CONFIG1, tag='test.metrics')
     Fluent::Test::OutputTestDriver.new(Fluent::GrowthForecastOutput, tag).configure(conf)
   end
@@ -278,6 +287,38 @@ class GrowthForecastOutputTest < Test::Unit::TestCase
     assert_equal 'field z', v[:name]
   end
 
+  # CONFIG_NON_KEEPALIVE = %[
+  #     gfapi_url http://127.0.0.1:5125/api/
+  #     service   service
+  #     section   metrics
+  #     name_keys field1,field2,otherfield
+  #     tag_for   name_prefix
+  #     keepalive false
+  # ]
+  def test_non_keepalive
+    d = create_driver(CONFIG_NON_KEEPALIVE, 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run
+
+    assert_equal 3, @posted.size
+    v1st = @posted[0]
+    v2nd = @posted[1]
+    v3rd = @posted[2]
+
+    assert_equal 50, v1st[:data][:number]
+    assert_equal 'gauge', v1st[:data][:mode]
+    assert_nil v1st[:auth]
+    assert_equal 'service', v1st[:service]
+    assert_equal 'metrics', v1st[:section]
+    assert_equal 'test.metrics_field1', v1st[:name]
+
+    assert_equal 20, v2nd[:data][:number]
+    assert_equal 'test.metrics_field2', v2nd[:name]
+
+    assert_equal 1, v3rd[:data][:number]
+    assert_equal 'test.metrics_otherfield', v3rd[:name]
+  end
+
   # setup / teardown for servers
   def setup
     Fluent::Test.setup
@@ -351,11 +392,11 @@ class GrowthForecastOutputTest < Test::Unit::TestCase
         end
       end
       cv.signal
-    } 
+    }
     mutex = Mutex.new
     mutex.synchronize {
       cv.wait(mutex)
-    } 
+    }
   end
 
   def test_dummy_server

metadata

@@ -1,62 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-growthforecast
 version: !ruby/object:Gem::Version
-  version: 0.1.6
-  prerelease: 
+  version: 0.2.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-08 00:00:00.000000000 Z
+date: 2013-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-config-placeholders
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: For GrowthForecast, see http://kazeburo.github.com/GrowthForecast/
@@ -80,27 +73,26 @@ files:
 - test/plugin/test_out_growthforecast.rb
 homepage: https://github.com/tagomoris/fluent-plugin-growthforecast
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fluentd output plugin to post numbers to GrowthForecast (by kazeburo)
 test_files:
 - test/helper.rb