fluent-plugin-growthforecast 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e4201bfd9823a61a81953e23c02a50ab12cf654
-  data.tar.gz: 6fcb119c643d549cd18e962207cad1c9c0d890c7
+  metadata.gz: ce309fd9162a5d30d1a24cfaed6c72feb607ecbe
+  data.tar.gz: 649824e492f0132967303c1d0f7400c1bdcab98c
 SHA512:
-  metadata.gz: 1a4f51b1fac0d9490ce105641baa1cf3ff49ce7d897eed6015153fcce9db804866ff19f443e0a73ad32919e9ed788feede39edce5d9824f9b1a7447eb0ac8ead
-  data.tar.gz: 05aff06da6206939ab35f19584507f1f453058bea42c4758337da7b998392b4af198966b7eacaa728da1f3ba2886981cc748855dfbba4b54b6b1f6032a299da5
+  metadata.gz: 2301b6ce097f7ff46d55a0e972574a5d8b0ba57f2555be7d575bc936b40cfa8f2c394e3d6c2196c3dc77c97339a3cf90c5dbbe8f60fc1d959ed52e393c6e9f72
+  data.tar.gz: 16de04d7a45a0810db0da6de6bb521082899cf488808baa60aa8cd1af2d20ba3693c57191b2bfc77170b82572d4d5357b6991a93a0a07768b09a32e5999bcdf6

data/README.md

@@ -82,6 +82,92 @@ Version v0.2.0 or later, this plugin uses HTTP connection keep-alive for a batch
 	  keepalive no
     </match>
 
+## Parameters
+
+* gfapi\_url (required)
+
+    The URL of a GrowthForecast API endpoint like `http://growth.forecast.local/api/`.
+    
+* tag\_for
+
+    Either of `name_prefix`, `section`, `service`, or `ignore`. Default is `name_prefix`. 
+
+    * `name_prefix` uses the tag name as a graph\_name prefix. 
+    * `section` uses the tag name as a section\_name.
+    * `service` uses the tag name as a service\_name.
+    * `ignore` uses the tag name for nothing.
+    
+* remove\_prefix
+
+    The prefix string which will be removed from the tag. This option would be useful using with the `tag_for` option.
+
+* service
+
+    The service\_name of graphs to create.
+
+* section
+
+    The section\_name of graphs to create.
+
+* name\_keys
+
+    Specify field names of the input record. Separate by , (comma).
+    The values of these fields are posted as numbers, and names of thease fields are used as parts of grame\_names. 
+    Either of `name_keys` or `name_key_pattern` is required. 
+
+* name\_key\_pattern
+
+    Specify the field names of the input record by a regular expression.
+    The values of these fields are posted as numbers, and names of thease fields are used as parts of grame\_names. 
+    Either of `name_keys` or `name_key_pattern` is required. 
+
+* graphs
+
+    You may use this option to specify graph names correspond to each of `name_keys`. Separate by , (comma). 
+    The number of graph names must be same with the number of `name_keys`.
+
+* mode
+
+    The graph mode (either of `gauge`, `count`, or `modified`). Just same as `mode` of GrowthForecast POST parameter. Default is `gauge`. 
+    
+* keepalive
+
+    Use a keepalive HTTP connection. Default is false.
+
+    NOTE: To effectively use this, you may need to give a parameter `max_keepalive_reqs` (default: 1) to Starlet in `growthforecast.pl`. 
+    
+* background_post
+
+    Post to GrowthForecast in background thread, without retries for failures (Default: false)
+
+* timeout
+
+    Read/Write timeout seconds (Default: 60)
+
+* retry
+
+    Do retry for HTTP request failures, or not. This feature will be set as false for `background_post yes` automatically. (Default: true)
+
+* ssl
+
+    Use SSL (https) or not. Default is false. 
+    
+* verify\_ssl
+
+    Do SSL verification or not. Default is false (ignore the SSL verification).
+
+* authentication
+
+    Specify `basic` if your GrowthForecast protected with basic authentication. Default is 'none' (no authentication).
+    
+* username
+
+    The username for authentication.
+
+* password
+
+    The password for authentication.
+
 ## TODO
 
 * patches welcome!

data/fluent-plugin-growthforecast.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-growthforecast"
-  gem.version       = "0.2.2"
+  gem.version       = "0.2.3"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd output plugin to post numbers to GrowthForecast (by kazeburo)}

data/lib/fluent/plugin/out_growthforecast.rb

@@ -11,6 +11,7 @@ class Fluent::GrowthForecastOutput < Fluent::Output
   config_param :gfapi_url, :string # growth.forecast.local/api/
   config_param :service, :string, :default => nil
   config_param :section, :string, :default => nil
+  config_param :graphs, :string, :default => nil
 
   config_param :ssl, :bool, :default => false
   config_param :verify_ssl, :bool, :default => false
@@ -23,6 +24,10 @@ class Fluent::GrowthForecastOutput < Fluent::Output
   config_param :remove_prefix, :string, :default => nil
   config_param :tag_for, :string, :default => 'name_prefix' # or 'ignore' or 'section' or 'service'
 
+  config_param :background_post, :bool, :default => false
+
+  config_param :timeout, :integer, :default => nil # default 60secs
+  config_param :retry, :bool, :default => true
   config_param :keepalive, :bool, :default => true
 
   config_param :authentication, :string, :default => nil # nil or 'none' or 'basic'
@@ -42,6 +47,10 @@ class Fluent::GrowthForecastOutput < Fluent::Output
     if not @name_keys.nil? and not @name_key_pattern.nil?
       raise Fluent::ConfigError, "cannot specify both of name_keys and name_key_pattern"
     end
+    if not @graphs.nil? and @name_keys.nil?
+      raise Fluent::ConfigError, "graphs must be specified with name_keys"
+    end
+
     if @name_keys
       @name_keys = @name_keys.split(',')
     end
@@ -49,6 +58,13 @@ class Fluent::GrowthForecastOutput < Fluent::Output
       @name_key_pattern = Regexp.new(@name_key_pattern)
     end
 
+    if @graphs
+      @graphs = @graphs.split(',')
+    end
+    if @name_keys and @graphs and @name_keys.size != @graphs.size
+      raise Fluent::ConfigError, "sizes of name_keys and graphs do not match"
+    end
+
     @mode = case @mode
             when 'count' then :count
             when 'modified' then :modified
@@ -84,12 +100,43 @@ class Fluent::GrowthForecastOutput < Fluent::Output
 
   def start
     super
+
+    @running = true
+    @thread = nil
+    @queue = nil
+    @mutex = nil
+    if @background_post
+      @mutex = Mutex.new
+      @queue = []
+      @thread = Thread.new(&method(:poster))
+    end
   end
 
   def shutdown
+    @running = false
+    @thread.join if @thread
     super
   end
 
+  def poster
+    while @running
+      if @queue.size < 1
+        sleep(0.2)
+        next
+      end
+
+      events = @mutex.synchronize {
+        es,@queue = @queue,[]
+        es
+      }
+      begin
+        post_events(events) if events.size > 0
+      rescue => e
+        $log.warn "HTTP POST Error occures to growthforecast server", :error_class => e.class, :error => e.message
+      end
+    end
+  end
+
   def format_url(tag, name)
     if @remove_prefix and
         ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
@@ -115,6 +162,10 @@ class Fluent::GrowthForecastOutput < Fluent::Output
 
   def http_connection(host, port)
     http = Net::HTTP.new(@resolver.getaddress(host), port)
+    if @timeout
+      http.open_timeout = @timeout
+      http.read_timeout = @timeout
+    end
     if @ssl
       http.use_ssl = true
       unless @verify_ssl
@@ -177,13 +228,23 @@ class Fluent::GrowthForecastOutput < Fluent::Output
     end
   end
 
+  def post_events(events)
+    if @keepalive
+      post_keepalive(events)
+    else
+      events.each do |event|
+        post(event[:tag], event[:name], event[:value])
+      end
+    end
+  end
+
   def emit(tag, es, chain)
     events = []
     if @name_keys
       es.each {|time,record|
-        @name_keys.each {|name|
-          if record[name]
-            events.push({:tag => tag, :name => name, :value => record[name]})
+        @name_keys.each_with_index {|name, i|
+          if value = record[name]
+            events.push({:tag => tag, :name => (@graphs ? @graphs[i] : name), :value => value})
           end
         }
       }
@@ -196,11 +257,16 @@ class Fluent::GrowthForecastOutput < Fluent::Output
         }
       }
     end
-    if @keepalive
-      post_keepalive(events)
+    if @thread
+      @mutex.synchronize do
+        @queue += events
+      end
     else
-      events.each do |event|
-        post(event[:tag], event[:name], event[:value])
+      begin
+        post_events(events)
+      rescue => e
+        $log.warn "HTTP POST Error occures to growthforecast server", :error_class => e.class, :error => e.message
+        raise if @retry
       end
     end
 

data/test/plugin/test_out_growthforecast.rb

@@ -55,6 +55,54 @@ class GrowthForecastOutputTest < Test::Unit::TestCase
       keepalive false
   ]
 
+  CONFIG_THREADING_KEEPALIVE = %[
+      gfapi_url http://127.0.0.1:5125/api/
+      service   service
+      section   metrics
+      name_keys field1,field2,otherfield
+      tag_for   name_prefix
+      background_post true
+      keepalive true
+      timeout   120
+  ]
+
+  CONFIG_THREADING_NON_KEEPALIVE = %[
+      gfapi_url http://127.0.0.1:5125/api/
+      service   service
+      section   metrics
+      name_keys field1,field2,otherfield
+      tag_for   name_prefix
+      background_post true
+      keepalive false
+  ]
+
+  CONFIG_BAD_GRAPHS = %[
+      gfapi_url http://127.0.0.1:5125/api/
+      service   service
+      section   metrics
+      name_keys field1,field2,otherfield
+      graphs    graph1,graph2
+      tag_for   name_prefix
+  ]
+
+  CONFIG_GRAPHS_WITHOUT_NAME_KEYS = %[
+      gfapi_url http://127.0.0.1:5125/api/
+      service   service
+      section   metrics
+      name_key_pattern ^(field|key)\\d+$
+      graphs    graph1,graph2
+      tag_for   name_prefix
+  ]
+
+  CONFIG_GRAPHS = %[
+      gfapi_url http://127.0.0.1:5125/api/
+      service   service
+      section   metrics
+      name_keys field1,field2,otherfield
+      graphs    graph1,graph2,othergraph
+      tag_for   name_prefix
+  ]
+
   def create_driver(conf=CONFIG1, tag='test.metrics')
     Fluent::Test::OutputTestDriver.new(Fluent::GrowthForecastOutput, tag).configure(conf)
   end
@@ -93,6 +141,9 @@ class GrowthForecastOutputTest < Test::Unit::TestCase
     assert_equal :modified, d.instance.mode
 
     assert_equal 'http://127.0.0.1:5125/api/service/data1/field1', d.instance.format_url('test.data1', 'field1')
+
+    assert_raise(Fluent::ConfigError) { d = create_driver(CONFIG_BAD_GRAPHS) }
+    assert_raise(Fluent::ConfigError) { d = create_driver(CONFIG_GRAPHS_WITHOUT_NAME_KEYS) }
   end
 
   # CONFIG1 = %[
@@ -319,6 +370,108 @@ class GrowthForecastOutputTest < Test::Unit::TestCase
     assert_equal 'test.metrics_otherfield', v3rd[:name]
   end
 
+  # CONFIG_THREADING_KEEPALIVE = %[
+  #     gfapi_url http://127.0.0.1:5125/api/
+  #     service   service
+  #     section   metrics
+  #     name_keys field1,field2,otherfield
+  #     tag_for   name_prefix
+  #     background_post true
+  #     keepalive true
+  #     timeout   120
+  # ]
+  def test_threading
+    d = create_driver(CONFIG_THREADING_KEEPALIVE, 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run
+    sleep 0.5 # wait internal posting thread loop
+
+    assert_equal 3, @posted.size
+    v1st = @posted[0]
+    v2nd = @posted[1]
+    v3rd = @posted[2]
+
+    assert_equal 50, v1st[:data][:number]
+    assert_equal 'gauge', v1st[:data][:mode]
+    assert_nil v1st[:auth]
+    assert_equal 'service', v1st[:service]
+    assert_equal 'metrics', v1st[:section]
+    assert_equal 'test.metrics_field1', v1st[:name]
+
+    assert_equal 20, v2nd[:data][:number]
+    assert_equal 'test.metrics_field2', v2nd[:name]
+
+    assert_equal 1, v3rd[:data][:number]
+    assert_equal 'test.metrics_otherfield', v3rd[:name]
+  end
+
+  # CONFIG_THREADING_NON_KEEPALIVE = %[
+  #     gfapi_url http://127.0.0.1:5125/api/
+  #     service   service
+  #     section   metrics
+  #     name_keys field1,field2,otherfield
+  #     tag_for   name_prefix
+  #     background_post true
+  #     keepalive false
+  # ]
+  def test_threading_non_keepalive
+    d = create_driver(CONFIG_THREADING_NON_KEEPALIVE, 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run
+    sleep 0.5 # wait internal posting thread loop
+
+    assert_equal 3, @posted.size
+    v1st = @posted[0]
+    v2nd = @posted[1]
+    v3rd = @posted[2]
+
+    assert_equal 50, v1st[:data][:number]
+    assert_equal 'gauge', v1st[:data][:mode]
+    assert_nil v1st[:auth]
+    assert_equal 'service', v1st[:service]
+    assert_equal 'metrics', v1st[:section]
+    assert_equal 'test.metrics_field1', v1st[:name]
+
+    assert_equal 20, v2nd[:data][:number]
+    assert_equal 'test.metrics_field2', v2nd[:name]
+
+    assert_equal 1, v3rd[:data][:number]
+    assert_equal 'test.metrics_otherfield', v3rd[:name]
+  end
+
+
+  # CONFIG_GRAPHS = %[
+  #     gfapi_url http://127.0.0.1:5125/api/
+  #     service   service
+  #     section   metrics
+  #     name_keys field1,field2,otherfield
+  #     graphs    graph1,graph2,othergraph
+  #     tag_for   name_prefix
+  # ]
+  def test_graphs
+    d = create_driver(CONFIG_GRAPHS, 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run
+
+    assert_equal 3, @posted.size
+    v1st = @posted[0]
+    v2nd = @posted[1]
+    v3rd = @posted[2]
+
+    assert_equal 50, v1st[:data][:number]
+    assert_equal 'gauge', v1st[:data][:mode]
+    assert_nil v1st[:auth]
+    assert_equal 'service', v1st[:service]
+    assert_equal 'metrics', v1st[:section]
+    assert_equal 'test.metrics_graph1', v1st[:name]
+
+    assert_equal 20, v2nd[:data][:number]
+    assert_equal 'test.metrics_graph2', v2nd[:name]
+
+    assert_equal 1, v3rd[:data][:number]
+    assert_equal 'test.metrics_othergraph', v3rd[:name]
+  end
+
   # setup / teardown for servers
   def setup
     Fluent::Test.setup

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-growthforecast
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-16 00:00:00.000000000 Z
+date: 2013-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -104,11 +104,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin to post numbers to GrowthForecast (by kazeburo)
 test_files:
 - test/helper.rb
 - test/plugin/test_out_growthforecast.rb
-has_rdoc: