fluent-plugin-yohoushi 0.0.5 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d33e56754f4290c20f7b06b38587f735fc86c2f
-  data.tar.gz: f5e963e6b9fa7bb2ff74c8c63d551906277da04f
+  metadata.gz: 65e9f7d306f6d8b262d7af0585fd52b898ed2c3b
+  data.tar.gz: d9caea2616664dd9d2bbffc6abefba2855d9d5ea
 SHA512:
-  metadata.gz: e7c1f30ae26ff1025657de6b2812a88e46785e510b69a3d0e98aca8a51065349a481c6551e23b643c4d6d6ec9db2e30218fcbd767ea89d50d4a7a64db9ac7a94
-  data.tar.gz: 667ae677f9de049f407561f5425c945748189581b53bd65282fac15546c261150658d2af240994519a4838c06db0650e155e384e5690799d08d23cbee62f1515
+  metadata.gz: 7f1e6c7d991049ad623219a997452a279aa0a105c9ea2353ea5d4549b3df0f92a2f74169e74e4619348cb74966a2c222ee1b692e290b5da501ca191313ef6832
+  data.tar.gz: 8ec234f0bc39e5c4df6a8f5804ccb29759fd256e96966ad3733730f3c870c4063972df958d557fa626c537a9a6bb095616984f0bdb925ebe0bc36decd5282a51

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.1.0 (2014/01/29)
+
+Changes:
+
+* fluent-plugin-yohoushi is now a buffer plugin
+
 ## 0.0.5 (2014/01/25)
 
 Fixes:

data/fluent-plugin-yohoushi.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-yohoushi"
-  s.version     = "0.0.5"
+  s.version     = "0.1.0"
   s.authors     = ["Naotoshi Seo"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-yohoushi"

data/lib/fluent/plugin/out_yohoushi.rb

@@ -1,227 +1,239 @@
-class Fluent::YohoushiOutput < Fluent::Output
-  Fluent::Plugin.register_output('yohoushi', self)
-
-  MAPPING_MAX_NUM = 20
-  KEY_MAX_NUM = 20
-
-  def initialize
-    super
-    require 'socket'
-    require 'multiforecast-client'
-    require 'yohoushi-client'
-  end
-
-  config_param :base_uri, :string, :default => nil
-  (1..MAPPING_MAX_NUM).each {|i| config_param "mapping#{i}".to_sym, :string, :default => nil }
-  config_param :key_pattern, :string, :default => nil
-  (1..KEY_MAX_NUM).each {|i| config_param "key#{i}".to_sym, :string, :default => nil }
-  config_param :enable_float_number, :bool, :default => false
-  config_param :mode, :default => :gauge do |val|
-    case val.downcase
-    when 'gauge'
-      :gauge
-    when 'count'
-      :count
-    when 'modified'
-      :modified
-    when 'derive'
-      :derive
-    else
-      raise Fluent::ConfigError, "stdout output output_type should be `gauge`, `count`, `modified`, or `derive`"
+module Fluent
+  class YohoushiOutput < BufferedOutput
+    Plugin.register_output('yohoushi', self)
+
+    MAPPING_MAX_NUM = 20
+    KEY_MAX_NUM = 20
+
+    def initialize
+      super
+      require 'socket'
+      require 'multiforecast-client'
+      require 'yohoushi-client'
     end
-  end
-  config_param :enable_ruby, :bool, :default => true # true for lower version compatibility
-
-  # for test
-  attr_reader :client
-  attr_reader :mapping
-  attr_reader :keys
-  attr_reader :key_pattern
-  attr_reader :key_pattern_path
-
-  def configure(conf)
-    super
-
-    if @base_uri
-      @client = Yohoushi::Client.new(@base_uri)
-    else
-      @mapping = {}
-      (1..MAPPING_MAX_NUM).each do |i|
-        next unless conf["mapping#{i}"]
-        from, to = conf["mapping#{i}"].split(/ +/, 2)
-        raise Fluent::ConfigError, "mapping#{i} does not contain 2 parameters" unless to
-        @mapping[from] = to
-      end
-      @client = MultiForecast::Client.new('mapping' => @mapping) unless @mapping.empty?
-    end
-    raise Fluent::ConfigError, "Either of `base_uri` or `mapping1` must be specified" unless @client
-
-    if @key_pattern
-      key_pattern, @key_pattern_path = @key_pattern.split(/ +/, 2)
-      raise Fluent::ConfigError, "key_pattern does not contain 2 parameters" unless @key_pattern_path
-      @key_pattern = Regexp.compile(key_pattern)
-    else
-      @keys = {}
-      (1..KEY_MAX_NUM).each do |i|
-        next unless conf["key#{i}"] 
-        key, path = conf["key#{i}"].split(/ +/, 2)
-        raise Fluent::ConfigError, "key#{i} does not contain 2 parameters" unless path
-        @keys[key] = path
+
+    config_param :base_uri, :string, :default => nil
+    (1..MAPPING_MAX_NUM).each {|i| config_param "mapping#{i}".to_sym, :string, :default => nil }
+    config_param :key_pattern, :string, :default => nil
+    (1..KEY_MAX_NUM).each {|i| config_param "key#{i}".to_sym, :string, :default => nil }
+    config_param :enable_float_number, :bool, :default => false
+    config_param :mode, :default => :gauge do |val|
+      case val.downcase
+      when 'gauge'
+        :gauge
+      when 'count'
+        :count
+      when 'modified'
+        :modified
+      when 'derive'
+        :derive
+      else
+        raise ConfigError, "stdout output output_type should be `gauge`, `count`, `modified`, or `derive`"
       end
     end
-    raise Fluent::ConfigError, "Either of `key_pattern` or `key1` must be specified" if (@key_pattern.nil? and @keys.empty?)
-
-    @placeholder_expander =
-      if @enable_ruby
-        # require utilities which would be used in ruby placeholders
-        require 'pathname'
-        require 'uri'
-        require 'cgi'
-        RubyPlaceholderExpander.new
+    config_param :enable_ruby, :bool, :default => true # true for lower version compatibility
+
+    # Override default parameters of Bufferedoutput options
+    config_param :buffer_type, :string, :default => 'memory'
+    config_param :flush_interval, :time, :default => 0 # we can not wait 1 minute to create 1 minute graphs (originally, 60)
+    config_param :try_flush_interval, :float, :default => 1 # we would be able to shorten more
+    config_param :retry_limit, :integer, :default => 1 # growthforecast requires a realtime post, so retry only once (originally, 17)
+    config_param :retry_wait, :time, :default => 1.0
+    config_param :max_retry_wait, :time, :default => nil
+    config_param :num_threads, :integer, :default => 1
+    config_param :queued_chunk_flush_interval, :time, :default => 1
+
+    # for test
+    attr_reader :client
+    attr_reader :mapping
+    attr_reader :keys
+    attr_reader :key_pattern
+    attr_reader :key_pattern_path
+
+    def configure(conf)
+      super
+
+      if @base_uri
+        @client = Yohoushi::Client.new(@base_uri)
       else
-        PlaceholderExpander.new
+        @mapping = {}
+        (1..MAPPING_MAX_NUM).each do |i|
+          next unless conf["mapping#{i}"]
+          from, to = conf["mapping#{i}"].split(/ +/, 2)
+          raise ConfigError, "mapping#{i} does not contain 2 parameters" unless to
+          @mapping[from] = to
+        end
+        @client = MultiForecast::Client.new('mapping' => @mapping) unless @mapping.empty?
       end
+      raise ConfigError, "Either of `base_uri` or `mapping1` must be specified" unless @client
 
-    @hostname = Socket.gethostname
-  rescue => e
-    raise Fluent::ConfigError, "#{e.class} #{e.message} #{e.backtrace.first}"
-  end
-
-  def start
-    super
-  end
-
-  def shutdown
-    super
-  end
-
-  def post(path, number)
-    if @enable_float_number
-      @client.post_graph(path, { 'number' => number.to_f, 'mode' => @mode.to_s })
-    else
-      @client.post_graph(path, { 'number' => number.to_i, 'mode' => @mode.to_s })
-    end
-  rescue => e
-    $log.warn "out_yohoushi: #{e.class} #{e.message} #{e.backtrace.first}"
-  end
-
-  def emit(tag, es, chain)
-    tag_parts = tag.split('.')
-    tag_prefix = tag_prefix(tag_parts)
-    tag_suffix = tag_suffix(tag_parts)
-    placeholders = {
-      'tag' => tag,
-      'tags' => tag_parts, # for lower compatibility
-      'tag_parts' => tag_parts,
-      'tag_prefix' => tag_prefix,
-      'tag_suffix' => tag_suffix,
-      'hostname' => @hostname,
-    }
-    if @key_pattern
-      es.each do |time, record|
-        record.each do |key, value|
-          next unless key =~ @key_pattern
-          placeholders['key'] = key
-          path = expand_placeholder(@key_pattern_path, time, record, placeholders)
-          post(path, value)
+      if @key_pattern
+        key_pattern, @key_pattern_path = @key_pattern.split(/ +/, 2)
+        raise ConfigError, "key_pattern does not contain 2 parameters" unless @key_pattern_path
+        @key_pattern = Regexp.compile(key_pattern)
+      else
+        @keys = {}
+        (1..KEY_MAX_NUM).each do |i|
+          next unless conf["key#{i}"] 
+          key, path = conf["key#{i}"].split(/ +/, 2)
+          raise ConfigError, "key#{i} does not contain 2 parameters" unless path
+          @keys[key] = path
         end
       end
-    else # keys
-      es.each do |time, record|
-        @keys.each do |key, path|
-          next unless value = record[key]
-          placeholders['key'] = key
-          path = expand_placeholder(path, time, record, placeholders)
-          post(path, value)
+      raise ConfigError, "Either of `key_pattern` or `key1` must be specified" if (@key_pattern.nil? and @keys.empty?)
+
+      @placeholder_expander =
+        if @enable_ruby
+          # require utilities which would be used in ruby placeholders
+          require 'pathname'
+          require 'uri'
+          require 'cgi'
+          RubyPlaceholderExpander.new
+        else
+          PlaceholderExpander.new
         end
-      end
+
+      @hostname = Socket.gethostname
+    rescue => e
+      raise ConfigError, "#{e.class} #{e.message} #{e.backtrace.first}"
     end
 
-    chain.next
-  rescue => e
-    $log.warn "out_yohoushi: #{e.class} #{e.message} #{e.backtrace.first}"
-  end
+    def start
+      super
+    end
 
-  def expand_placeholder(value, time, record, opts)
-    @placeholder_expander.prepare_placeholders(time, record, opts)
-    @placeholder_expander.expand(value)
-  end
+    def shutdown
+      super
+    end
 
-  def tag_prefix(tag_parts)
-    return [] if tag_parts.empty?
-    tag_prefix = [tag_parts.first]
-    1.upto(tag_parts.size-1).each do |i|
-      tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}"
+    def post(path, number)
+      if @enable_float_number
+        @client.post_graph(path, { 'number' => number.to_f, 'mode' => @mode.to_s })
+      else
+        @client.post_graph(path, { 'number' => number.to_i, 'mode' => @mode.to_s })
+      end
+    rescue => e
+      $log.warn "out_yohoushi: #{e.class} #{e.message} #{e.backtrace.first}"
     end
-    tag_prefix
-  end
 
-  def tag_suffix(tag_parts)
-    return [] if tag_parts.empty?
-    rev_tag_parts = tag_parts.reverse
-    rev_tag_suffix = [rev_tag_parts.first]
-    1.upto(tag_parts.size-1).each do |i|
-      rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}"
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
     end
-    rev_tag_suffix.reverse
-  end
 
-  class PlaceholderExpander
-    attr_reader :placeholders
-
-    def prepare_placeholders(time, record, opts)
-      placeholders = { '${time}' => Time.at(time).to_s }
-      record.each {|key, val| placeholders.store("${#{key}}", val) }
-
-      opts.each do |key, val|
-        if val.kind_of?(Array)
-          size = val.size
-          val.each_with_index { |t, idx|
-            placeholders.store("${#{key}[#{idx}]}", t)
-            placeholders.store("${#{key}[#{idx-size}]}", t) # support [-1]
-          }
-        else # string, interger, float, and others?
-          placeholders["${#{key}}"] = val
+    def write(chunk)
+      chunk.msgpack_each do |tag, time, record|
+        tag_parts = tag.split('.')
+        tag_prefix = tag_prefix(tag_parts)
+        tag_suffix = tag_suffix(tag_parts)
+        placeholders = {
+          'tag' => tag,
+          'tags' => tag_parts, # for lower compatibility
+          'tag_parts' => tag_parts,
+          'tag_prefix' => tag_prefix,
+          'tag_suffix' => tag_suffix,
+          'hostname' => @hostname,
+        }
+        if @key_pattern
+          record.each do |key, value|
+            next unless key =~ @key_pattern
+            placeholders['key'] = key
+            path = expand_placeholder(@key_pattern_path, time, record, placeholders)
+            post(path, value)
+          end
+        else # keys
+          @keys.each do |key, path|
+            next unless value = record[key]
+            placeholders['key'] = key
+            path = expand_placeholder(path, time, record, placeholders)
+            post(path, value)
+          end
         end
       end
+    rescue => e
+      $log.warn "out_yohoushi: #{e.class} #{e.message} #{e.backtrace.first}"
+    end
 
-      @placeholders = placeholders
+    def expand_placeholder(value, time, record, opts)
+      @placeholder_expander.prepare_placeholders(time, record, opts)
+      @placeholder_expander.expand(value)
     end
 
-    def expand(str)
-      str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
-        $log.warn "record_reformer: unknown placeholder `#{$1}` found" unless @placeholders.include?($1)
-        @placeholders[$1]
-      }
+    def tag_prefix(tag_parts)
+      return [] if tag_parts.empty?
+      tag_prefix = [tag_parts.first]
+      1.upto(tag_parts.size-1).each do |i|
+        tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}"
+      end
+      tag_prefix
     end
-  end
 
-  class RubyPlaceholderExpander
-    attr_reader :placeholders
-
-    # Get placeholders as a struct
-    #
-    # @param [Time]   time        the time
-    # @param [Hash]   record      the record
-    # @param [Hash]   opts        others
-    def prepare_placeholders(time, record, opts)
-      struct = UndefOpenStruct.new(record)
-      struct.time = Time.at(time)
-      opts.each {|key, val| struct.__send__("#{key}=", val) }
-      @placeholders = struct
+    def tag_suffix(tag_parts)
+      return [] if tag_parts.empty?
+      rev_tag_parts = tag_parts.reverse
+      rev_tag_suffix = [rev_tag_parts.first]
+      1.upto(tag_parts.size-1).each do |i|
+        rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}"
+      end
+      rev_tag_suffix.reverse
     end
 
-    # Replace placeholders in a string
-    #
-    # @param [String] str         the string to be replaced
-    def expand(str)
-      str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
-      eval "\"#{str}\"", @placeholders.instance_eval { binding }
+    class PlaceholderExpander
+      attr_reader :placeholders
+
+      def prepare_placeholders(time, record, opts)
+        placeholders = { '${time}' => Time.at(time).to_s }
+        record.each {|key, value| placeholders.store("${#{key}}", value) }
+
+        opts.each do |key, value|
+          if value.kind_of?(Array) # tag_parts, etc
+            size = value.size
+            value.each_with_index { |v, idx|
+              placeholders.store("${#{key}[#{idx}]}", v)
+              placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1]
+            }
+          else # string, interger, float, and others?
+            placeholders.store("${#{key}}", value)
+          end
+        end
+
+        @placeholders = placeholders
+      end
+
+      def expand(str)
+        str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
+          $log.warn "record_reformer: unknown placeholder `#{$1}` found" unless @placeholders.include?($1)
+          @placeholders[$1]
+        }
+      end
     end
 
-    class UndefOpenStruct < OpenStruct
-      (Object.instance_methods).each do |m|
-        undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+    class RubyPlaceholderExpander
+      attr_reader :placeholders
+
+      # Get placeholders as a struct
+      #
+      # @param [Time]   time        the time
+      # @param [Hash]   record      the record
+      # @param [Hash]   opts        others
+      def prepare_placeholders(time, record, opts)
+        struct = UndefOpenStruct.new(record)
+        struct.time = Time.at(time)
+        opts.each {|key, value| struct.__send__("#{key}=", value) }
+        @placeholders = struct
+      end
+
+      # Replace placeholders in a string
+      #
+      # @param [String] str         the string to be replaced
+      def expand(str)
+        str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
+        eval "\"#{str}\"", @placeholders.instance_eval { binding }
+      end
+
+      class UndefOpenStruct < OpenStruct
+        (Object.instance_methods).each do |m|
+          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+        end
       end
     end
   end

data/spec/out_yohoushi_spec.rb

@@ -19,9 +19,8 @@ describe Fluent::YohoushiOutput do
   let(:yohoushi_base_uri) { 'http://localhost:4804' }
   let(:growthforecast_base_uri) { 'http://localhost:5125' }
   let(:tag) { 'test' }
-  let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::YohoushiOutput, tag).configure(config) }
+  let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::YohoushiOutput, tag).configure(config) }
   let(:instance) { driver.instance }
-  let(:emit) { driver.run { messages.each {|message| driver.emit(message, time) } } }
 
   describe 'test configure' do
     context "empty" do
@@ -80,6 +79,7 @@ describe Fluent::YohoushiOutput do
   end
 
   describe 'test emit' do
+    let(:emit) { d = driver; messages.each {|message| d.emit(message, time) }; d.run; }
     let(:time) { Time.now.to_i }
     let(:messages) do
       [
@@ -144,7 +144,7 @@ describe Fluent::YohoushiOutput do
     let(:tag) { 'fluent.error' }
     let(:time) { Time.now.to_i }
     let(:record) { { 'foo_count' => "1" } }
-    let(:emit) { driver.run { driver.emit(record, time) } }
+    let(:emit) { d = driver; d.emit(record, time); d.run; }
     let(:expected_path)  { "/fluent/error/fluent.error/1/foo_count/#{Time.at(time)}" }
     let(:expected_value) { '1' }
     before { driver.instance.should_receive(:post).with(expected_path, expected_value) }
@@ -173,7 +173,7 @@ describe Fluent::YohoushiOutput do
     let(:tag) { 'fluent.error' }
     let(:time) { Time.now.to_i }
     let(:record) { { 'foo_count' => "1" } }
-    let(:emit) { driver.run { driver.emit(record, time) } }
+    let(:emit) { d = driver; d.emit(record, time); d.run; }
     let(:expected_path)  { "/fluent/error/fluent.error/1/foo_count/#{Time.at(time)}" }
     let(:expected_value) { '1' }
     before { driver.instance.should_receive(:post).with(expected_path, expected_value) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-yohoushi
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-25 00:00:00.000000000 Z
+date: 2014-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd