fluentd 0.12.12 → 0.12.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c5d20fa9a07887680fe5d8b7b5e6982c7abf515
-  data.tar.gz: a8db7dc9f26fcd657e02c99a457bc00c66757176
+  metadata.gz: e8d2b210d487f331113e6ba6be6f9607d3cb0129
+  data.tar.gz: 83237ccc84a3af2e52d8824fdda9bc5ee14485a8
 SHA512:
-  metadata.gz: df6783a7ae988dba16ae804cafd1750af435cbd3c89c8bce2ead550ece11addf1f0539637bcf119510d87db732eeb4f9ee58e97b03c5cce5e7437bc5af9a028b
-  data.tar.gz: b2a2c418a054dedcb654c6fb2ca93e568c39651774bb14bfe2465a630533a7108faa5b9414a228b4c0e188b850212a4622ea83f44766db26dac52d29dffedd24
+  metadata.gz: 206acfb5ddab70b8c7b20b1b36c25a62f827c899c851bbbb98484159309184266597cbebd691c53cb23aa5fac7a6e77ebcbe1611ada8bf720583ba93ed26faf6
+  data.tar.gz: b5381b0ec5c92f33a30e021db328767582aa2753d651695073a27c84099c4cb9445c127a0dc9a31350dd441a1ad53d05ef3fe79d488d644faa0a228150dfe706

data/ChangeLog

@@ -1,5 +1,23 @@
 # v0.12
 
+## Release 0.12.13 - 2015/07/09
+
+### New features / Enhancement
+
+* out_stdout: Support formatter plugins in out_stdout
+  https://github.com/fluent/fluentd/pull/617
+* filter_record_transformer: Add "auto_typecast" parameter
+  https://github.com/fluent/fluentd/pull/620
+* out_forward: Support DNS round robin
+  https://github.com/fluent/fluentd/pull/625
+* out_forward: Support heartbeat_type none to disable heartbeat
+  https://github.com/fluent/fluentd/pull/626
+
+### Bug fixes
+
+* config: Fix secret option to support config_section
+  https://github.com/fluent/fluentd/pull/628
+
 ## Release 0.12.12 - 2015/06/12
 
 ### New features / Enhancement

data/example/in_forward.conf

@@ -0,0 +1,7 @@
+<source>
+  type forward
+</source>
+
+<match test>
+  type stdout
+</match>

data/example/out_forward.conf

@@ -1,5 +1,6 @@
 <source>
-  type forward
+  type dummy
+  tag test
 </source>
 
 <match test>
@@ -7,18 +8,22 @@
 
   <server>
     # first server
+    host localhost
     port 24224
   </server>
   <server>
     # second server
+    host localhost
     port 24225
   </server>
   <server>
     # second server
+    host localhost
     port 24226
     standby
   </server>
 
+  flush_interval 1
   send_timeout 60
   heartbeat_type udp
   heartbeat_interval 1

data/lib/fluent/config/element.rb

@@ -118,11 +118,12 @@ module Fluent
       end
 
       def to_masked_element
-        element = Element.new(@name, @arg, {}, @elements, @unused)
+        new_elems = @elements.map { |e| e.to_masked_element }
+        new_elem = Element.new(@name, @arg, {}, new_elems, @unused)
         each_pair { |k, v|
-          element[k] = secret_param?(k) ? 'xxxxxx' : v
+          new_elem[k] = secret_param?(k) ? 'xxxxxx' : v
         }
-        element
+        new_elem
       end
 
       def secret_param?(key)

data/lib/fluent/config/section.rb

@@ -121,6 +121,10 @@ module Fluent
         proxy.sections.each do |name, subproxy|
           varname = subproxy.param_name.to_sym
           elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+          # set subproxy for secret option
+          elements.each { |element|
+            element.corresponding_proxies << subproxy
+          }
 
           if subproxy.required? && elements.size < 1
             logger.error "config error in:\n#{conf}"

data/lib/fluent/configurable.rb

@@ -45,12 +45,10 @@ module Fluent
 
     def configure(conf)
       @config = conf
-      if class_name = self.class.name # Class.new in tests returns nil so it should be skipped.
-        @config.corresponding_proxies << self.class.configure_proxy(class_name)
-      end
 
       logger = self.respond_to?(:log) ? log : $log
       proxy = self.class.merged_configure_proxy
+      conf.corresponding_proxies << proxy
 
       root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger)
       @config_root_section = root

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -30,6 +30,7 @@ module Fluent
     config_param :renew_record, :bool, :default => false
     config_param :renew_time_key, :string, :default => nil
     config_param :enable_ruby, :bool, :default => false
+    config_param :auto_typecast, :bool, :default => false # false for lower version compatibility
 
     def configure(conf)
       super
@@ -52,15 +53,19 @@ module Fluent
         @keep_keys = @keep_keys.split(',')
       end
 
+      placeholder_expander_params = {
+        :log           => log,
+        :auto_typecast => @auto_typecast,
+      }
       @placeholder_expander =
         if @enable_ruby
           # require utilities which would be used in ruby placeholders
           require 'pathname'
           require 'uri'
           require 'cgi'
-          RubyPlaceholderExpander.new(log)
+          RubyPlaceholderExpander.new(placeholder_expander_params)
         else
-          PlaceholderExpander.new(log)
+          PlaceholderExpander.new(placeholder_expander_params)
         end
 
       @hostname = Socket.gethostname
@@ -124,7 +129,7 @@ module Fluent
       elsif value.is_a?(Hash)
         new_value = {}
         value.each_pair do |k, v|
-          new_value[@placeholder_expander.expand(k)] = expand_placeholders(v)
+          new_value[@placeholder_expander.expand(k, true)] = expand_placeholders(v)
         end
       elsif value.is_a?(Array)
         new_value = []
@@ -159,8 +164,9 @@ module Fluent
     class PlaceholderExpander
       attr_reader :placeholders, :log
 
-      def initialize(log)
-        @log = log
+      def initialize(params)
+        @log = params[:log]
+        @auto_typecast = params[:auto_typecast]
       end
 
       def prepare_placeholders(time, record, opts)
@@ -182,19 +188,34 @@ module Fluent
         @placeholders = placeholders
       end
 
-      def expand(str)
+      def expand(str, force_stringify=false)
+        if @auto_typecast and !force_stringify
+          single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
+          if single_placeholder_matched
+            log_unknown_placeholder($1)
+            return @placeholders[single_placeholder_matched[1]]
+          end
+        end
         str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
-          log.warn "unknown placeholder `#{$1}` found" unless @placeholders.include?($1)
+          log_unknown_placeholder($1)
           @placeholders[$1]
         }
       end
+
+      private
+      def log_unknown_placeholder(placeholder)
+        unless @placeholders.include?(placeholder)
+          log.warn "unknown placeholder `#{placeholder}` found"
+        end
+      end
     end
 
     class RubyPlaceholderExpander
       attr_reader :placeholders, :log
 
-      def initialize(log)
-        @log = log
+      def initialize(params)
+        @log = params[:log]
+        @auto_typecast = params[:auto_typecast]
       end
 
       # Get placeholders as a struct
@@ -209,8 +230,15 @@ module Fluent
         @placeholders = struct
       end
 
-      def expand(str)
-        interpolated = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..} 
+      def expand(str, force_stringify=false)
+        if @auto_typecast and !force_stringify
+          single_placeholder_matched = str.match(/\A\${([^}]+)}\z/)
+          if single_placeholder_matched
+            code = single_placeholder_matched[1]
+            return eval code, @placeholders.instance_eval { binding }
+          end
+        end
+        interpolated = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
         eval "\"#{interpolated}\"", @placeholders.instance_eval { binding }
       rescue => e
         log.warn "failed to expand `#{str}`", :error_class => e.class, :error => e.message

data/lib/fluent/plugin/out_forward.rb

@@ -43,8 +43,10 @@ module Fluent
         :tcp
       when 'udp'
         :udp
+      when 'none'
+        :none
       else
-        raise ConfigError, "forward output heartbeat type should be 'tcp' or 'udp'"
+        raise ConfigError, "forward output heartbeat type should be 'tcp', 'udp', or 'none'"
       end
     end
     config_param :heartbeat_interval, :time, :default => 1
@@ -111,7 +113,12 @@ module Fluent
 
         node_conf = NodeConfig.new(name, host, port, weight, standby, failure,
           @phi_threshold, recover_sample_size, @expire_dns_cache, @phi_failure_detector)
-        @nodes << Node.new(log, node_conf)
+
+        if @heartbeat_type == :none
+          @nodes << NoneHeartbeatNode.new(log, node_conf)
+        else
+          @nodes << Node.new(log, node_conf)
+        end
         log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight, :plugin_id=>plugin_id
       }
     end
@@ -123,32 +130,36 @@ module Fluent
       rebuild_weight_array
       @rr = 0
 
-      @loop = Coolio::Loop.new
+      unless @heartbeat_type == :none
+        @loop = Coolio::Loop.new
 
-      if @heartbeat_type == :udp
-        # assuming all hosts use udp
-        @usock = SocketUtil.create_udp_socket(@nodes.first.host)
-        @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
-        @hb = HeartbeatHandler.new(@usock, method(:on_heartbeat))
-        @loop.attach(@hb)
-      end
+        if @heartbeat_type == :udp
+          # assuming all hosts use udp
+          @usock = SocketUtil.create_udp_socket(@nodes.first.host)
+          @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
+          @hb = HeartbeatHandler.new(@usock, method(:on_heartbeat))
+          @loop.attach(@hb)
+        end
 
-      @timer = HeartbeatRequestTimer.new(@heartbeat_interval, method(:on_timer))
-      @loop.attach(@timer)
+        @timer = HeartbeatRequestTimer.new(@heartbeat_interval, method(:on_timer))
+        @loop.attach(@timer)
 
-      @thread = Thread.new(&method(:run))
+        @thread = Thread.new(&method(:run))
+      end
     end
 
     def shutdown
       @finished = true
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @thread.join
+      if @loop
+        @loop.watchers.each {|w| w.detach }
+        @loop.stop
+      end
+      @thread.join if @thread
       @usock.close if @usock
     end
 
     def run
-      @loop.run
+      @loop.run if @loop
     rescue
       log.error "unexpected error", :error=>$!.to_s
       log.error_backtrace
@@ -460,9 +471,8 @@ module Fluent
       end
 
       def resolve_dns!
-        @sockaddr = Socket.pack_sockaddr_in(@port, @host)
-        port, resolved_host = Socket.unpack_sockaddr_in(@sockaddr)
-        return resolved_host
+        # sample to support dns round robin
+        Socket.getaddrinfo(@host, @port, nil, Socket::SOCK_STREAM).sample[3]
       end
       private :resolve_dns!
 
@@ -515,6 +525,21 @@ module Fluent
       end
     end
 
+    # Override Node to disable heartbeat
+    class NoneHeartbeatNode < Node
+      def available?
+        true
+      end
+
+      def tick
+        false
+      end
+
+      def heartbeat(detect=true)
+        true
+      end
+    end
+
     class FailureDetector
       PHI_FACTOR = 1.0 / Math.log(10.0)
       SAMPLE_SIZE = 1000

data/lib/fluent/plugin/out_stdout.rb

@@ -18,30 +18,17 @@ module Fluent
   class StdoutOutput < Output
     Plugin.register_output('stdout', self)
 
-    OUTPUT_PROCS = {
-      :json => Proc.new {|record| Yajl.dump(record) },
-      :hash => Proc.new {|record| record.to_s },
-    }
-
-    config_param :output_type, :default => :json do |val|
-      case val.downcase
-      when 'json'
-        :json
-      when 'hash'
-        :hash
-      else
-        raise ConfigError, "stdout output output_type should be 'json' or 'hash'"
-      end
-    end
+    config_param :output_type, :default => 'json'
 
     def configure(conf)
       super
-      @output_proc = OUTPUT_PROCS[@output_type]
+      @formatter = Plugin.new_formatter(@output_type)
+      @formatter.configure(conf)
     end
 
     def emit(tag, es, chain)
       es.each {|time,record|
-        $log.write "#{Time.at(time).localtime} #{tag}: #{@output_proc.call(record)}\n"
+        $log.write "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
       }
       $log.flush
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.12'
+  VERSION = '0.12.13'
 
 end

data/test/config/test_configurable.rb

@@ -152,6 +152,11 @@ module ConfigurableSpec
 
     config_param :normal_param, :string
     config_param :secret_param, :string, :secret => true
+
+    config_section :section  do
+      config_param :normal_param2, :string
+      config_param :secret_param2, :string, :secret => true
+    end
   end
 end
 
@@ -686,8 +691,14 @@ module Fluent::Config
     end
 
     sub_test_case ':secret option' do
+      def e(name, arg = '', attrs = {}, elements = [])
+        attrs_str_keys = {}
+        attrs.each { |key, value| attrs_str_keys[key.to_s] = value }
+        Fluent::Config::Element.new(name, arg, attrs_str_keys, elements)
+      end
+
       setup do
-        @conf = Fluent::Config::Element.new('ROOT', '', {'normal_param' => 'param1', 'secret_param' => 'param2'}, [])
+        @conf = e('ROOT', '', {'normal_param' => 'normal', 'secret_param' => 'secret'}, [e('section', '', {'normal_param2' => 'normal', 'secret_param2' => 'secret'} )])
         @example = ConfigurableSpec::Example5.new
         @example.configure(@conf)
       end
@@ -700,16 +711,22 @@ module Fluent::Config
       end
 
       test 'config returns masked configuration' do
-        @example.config.each_pair { |key, value|
+        conf = @example.config
+        conf.each_pair { |key, value|
           assert_secret_param(key, value)
         }
+        conf.elements.each { |element|
+          element.each_pair { |key, value|
+            assert_secret_param(key, value)
+          }
+        }
       end
 
       def assert_secret_param(key, value)
         case key
-        when 'normal_param'
-          assert_equal 'param1', value
-        when 'secret_param'
+        when 'normal_param', 'normal_param2'
+          assert_equal 'normal', value
+        when 'secret_param', 'secret_param2'
           assert_equal 'xxxxxx', value
         end
       end

data/test/plugin/test_filter_record_transformer.rb

@@ -172,7 +172,12 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       yield d if block_given?
       d.run {
         msgs.each do |msg|
-          d.emit({'eventType0' => 'bar', 'message' => msg}, @time)
+          record = {
+            'eventType0' => 'bar',
+            'message'    => msg,
+          }
+          record = record.merge(msg) if msg.is_a?(Hash)
+          d.emit(record, @time)
         end
       }.filtered
     end
@@ -325,6 +330,112 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           assert_equal({@hostname=>'hostname',"foo.#{@tag}"=>'tag'}, r)
         end
       end
+
+      test "disabled typecasting of values with enable_ruby #{enable_ruby}" do
+        config = %[
+          auto_typecast false
+          enable_ruby #{enable_ruby}
+          <record>
+            single      ${source}
+            multiple    ${source}${source}
+            with_prefix prefix-${source}
+            with_suffix ${source}-suffix
+          </record>
+        ]
+        msgs = [
+          { "source" => "string" },
+          { "source" => 123 },
+          { "source" => [1, 2] },
+          { "source" => {a:1, b:2} },
+          { "source" => nil },
+        ]
+        expected_results = [
+          { :single      => "string",
+            :multiple    => "stringstring",
+            :with_prefix => "prefix-string",
+            :with_suffix => "string-suffix" },
+          { :single      => 123.to_s,
+            :multiple    => "#{123.to_s}#{123.to_s}",
+            :with_prefix => "prefix-#{123.to_s}",
+            :with_suffix => "#{123.to_s}-suffix" },
+          { :single      => [1, 2].to_s,
+            :multiple    => "#{[1, 2].to_s}#{[1, 2].to_s}",
+            :with_prefix => "prefix-#{[1, 2].to_s}",
+            :with_suffix => "#{[1, 2].to_s}-suffix" },
+          { :single      => {a:1, b:2}.to_s,
+            :multiple    => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
+            :with_prefix => "prefix-#{{a:1, b:2}.to_s}",
+            :with_suffix => "#{{a:1, b:2}.to_s}-suffix" },
+          { :single      => nil.to_s,
+            :multiple    => "#{nil.to_s}#{nil.to_s}",
+            :with_prefix => "prefix-#{nil.to_s}",
+            :with_suffix => "#{nil.to_s}-suffix" },
+        ]
+        actual_results = []
+        es = emit(config, msgs)
+        es.each_with_index do |(t, r), i|
+          actual_results << {
+            :single      => r["single"],
+            :multiple    => r["multiple"],
+            :with_prefix => r["with_prefix"],
+            :with_suffix => r["with_suffix"],
+          }
+        end
+        assert_equal(expected_results, actual_results)
+      end
+
+      test "enabled typecasting of values with enable_ruby #{enable_ruby}" do
+        config = %[
+          auto_typecast yes
+          enable_ruby #{enable_ruby}
+          <record>
+            single      ${source}
+            multiple    ${source}${source}
+            with_prefix prefix-${source}
+            with_suffix ${source}-suffix
+          </record>
+        ]
+        msgs = [
+          { "source" => "string" },
+          { "source" => 123 },
+          { "source" => [1, 2] },
+          { "source" => {a:1, b:2} },
+          { "source" => nil },
+        ]
+        expected_results = [
+          { :single      => "string",
+            :multiple    => "stringstring",
+            :with_prefix => "prefix-string",
+            :with_suffix => "string-suffix" },
+          { :single      => 123,
+            :multiple    => "#{123.to_s}#{123.to_s}",
+            :with_prefix => "prefix-#{123.to_s}",
+            :with_suffix => "#{123.to_s}-suffix" },
+          { :single      => [1, 2],
+            :multiple    => "#{[1, 2].to_s}#{[1, 2].to_s}",
+            :with_prefix => "prefix-#{[1, 2].to_s}",
+            :with_suffix => "#{[1, 2].to_s}-suffix" },
+          { :single      => {a:1, b:2},
+            :multiple    => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
+            :with_prefix => "prefix-#{{a:1, b:2}.to_s}",
+            :with_suffix => "#{{a:1, b:2}.to_s}-suffix" },
+          { :single      => nil,
+            :multiple    => "#{nil.to_s}#{nil.to_s}",
+            :with_prefix => "prefix-#{nil.to_s}",
+            :with_suffix => "#{nil.to_s}-suffix" },
+        ]
+        actual_results = []
+        es = emit(config, msgs)
+        es.each_with_index do |(t, r), i|
+          actual_results << {
+            :single      => r["single"],
+            :multiple    => r["multiple"],
+            :with_prefix => r["with_prefix"],
+            :with_suffix => r["with_suffix"],
+          }
+        end
+        assert_equal(expected_results, actual_results)
+      end
     end
 
     test 'unknown placeholder (enable_ruby no)' do

data/test/plugin/test_out_forward.rb

@@ -59,6 +59,11 @@ class ForwardOutputTest < Test::Unit::TestCase
     assert_equal :tcp, d.instance.heartbeat_type
   end
 
+  def test_configure_none_heartbeat
+    d = create_driver(CONFIG + "\nheartbeat_type none")
+    assert_equal :none, d.instance.heartbeat_type
+  end
+
   def test_phi_failure_detector
     d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
     node = d.instance.nodes.first
@@ -327,6 +332,20 @@ class ForwardOutputTest < Test::Unit::TestCase
     }.configure(conf).inject_router()
   end
 
+  def test_heartbeat_type_none
+    d = create_driver(CONFIG + "\nheartbeat_type none")
+    node = d.instance.nodes.first
+    assert_equal Fluent::ForwardOutput::NoneHeartbeatNode, node.class
+
+    d.instance.start
+    assert_nil d.instance.instance_variable_get(:@loop)   # no HeartbeatHandler, or HeartbeatRequestTimer
+    assert_nil d.instance.instance_variable_get(:@thread) # no HeartbeatHandler, or HeartbeatRequestTimer
+
+    stub(node.failure).phi { raise 'Should not be called' }
+    node.tick
+    assert_equal node.available, true
+  end
+
   class DummyEngineDriver < Fluent::Test::TestDriver
     def initialize(klass, &block)
       super(klass, &block)

data/test/plugin/test_out_stdout.rb

@@ -15,15 +15,15 @@ class StdoutOutputTest < Test::Unit::TestCase
 
   def test_configure
     d = create_driver
-    assert_equal :json, d.instance.output_type
+    assert_equal 'json', d.instance.output_type
   end
 
   def test_configure_output_type
     d = create_driver(CONFIG + "\noutput_type json")
-    assert_equal :json, d.instance.output_type
+    assert_equal 'json', d.instance.output_type
 
     d = create_driver(CONFIG + "\noutput_type hash")
-    assert_equal :hash, d.instance.output_type
+    assert_equal 'hash', d.instance.output_type
 
     assert_raise(Fluent::ConfigError) do
       d = create_driver(CONFIG + "\noutput_type foo")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.12
+  version: 0.12.13
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-12 00:00:00.000000000 Z
+date: 2015-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -292,6 +292,7 @@ files:
 - bin/fluent-gem
 - bin/fluentd
 - example/filter_stdout.conf
+- example/in_forward.conf
 - example/in_http.conf
 - example/in_syslog.conf
 - example/in_tail.conf