fluentd 0.12.7 → 0.12.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 34615d5421761b2b42cd9831d2d87a96ac401b9d
-  data.tar.gz: 49ba7a22424ee071c07c166e35a5bff49720327b
+  metadata.gz: 67639a7dc51e74c0269b78897487a90bd0acf29b
+  data.tar.gz: 43fd091c294a06a0d88eea360a7efd5b215d5577
 SHA512:
-  metadata.gz: d71f71d6e0f4267836f6f683698a9dbe12c5447dd95a19a1a251bde03e058913a6b32de0a7bdad98783efb198660cf5a5dfebe76ec5e60ebf19d698654ded6b6
-  data.tar.gz: 6a09feb33ef26f30c0901d3f0e2e6b046d4a837b1679c69b7c87abf12f2de2cd5ce3a88ff86a220c81ee5a0e0cd8fcb5aea9d9899aa6b757be24ee79d63df2a3
+  metadata.gz: 1357f9b5a372172f9e6441dafdb106d8fa2e6756d3bb23c54b0d16c1e8888fc97e75b5ebbf94aac7ae589d48d42af6b08968653de0309cc8a9efe6b2a9ade81d
+  data.tar.gz: 00d88127b76af84ba10b1830c0be4861047a571cb953dfd63c7887d145ca8752fb8a4a978b586159918356ee57f6b7b00190295d9a6509467bd41791e8b63dfb

data/ChangeLog

@@ -1,5 +1,23 @@
 # v0.12
 
+## Release 0.12.8 - 2015/04/22
+
+### New features / Enhancement
+
+* output: Support millisecond for try_flush_interval and queued_chunk_flush_interval
+  https://github.com/fluent/fluentd/pull/568
+* filter_record_transformer: Support fields which start with @
+  https://github.com/fluent/fluentd/pull/574
+* config: Add final attribute to prevent config overwritten by subclass
+  https://github.com/fluent/fluentd/pull/583
+
+### Bug fixes
+
+* config: fix embedded code parsing
+  https://github.com/fluent/fluentd/pull/564
+* out_forward: Resend chunks on dropped connection when wait ACK
+  https://github.com/fluent/fluentd/pull/580
+
 ## Release 0.12.7 - 2015/03/22
 
 ### New features / Enhancement

data/lib/fluent/config/configure_proxy.rb

@@ -17,7 +17,7 @@
 module Fluent
   module Config
     class ConfigureProxy
-      attr_accessor :name, :param_name, :required, :multi, :alias, :argument, :params, :defaults, :sections
+      attr_accessor :name, :final, :param_name, :required, :multi, :alias, :argument, :params, :defaults, :sections
       # config_param :desc, :string, :default => '....'
       # config_set_default :buffer_type, :memory
       #
@@ -38,6 +38,7 @@ module Fluent
 
       def initialize(name, opts = {})
         @name = name.to_sym
+        @final = opts.fetch(:final, false)
 
         @param_name = (opts[:param_name] || @name).to_sym
         @required = opts[:required]
@@ -58,7 +59,13 @@ module Fluent
         @multi.nil? ? true : @multi
       end
 
+      def final?
+        @final
+      end
+
       def merge(other) # self is base class, other is subclass
+        return merge_for_finalized(other) if self.final?
+
         options = {
           param_name: other.param_name,
           required: (other.required.nil? ? self.required : other.required),
@@ -69,7 +76,52 @@ module Fluent
         merged.argument = other.argument || self.argument
         merged.params = self.params.merge(other.params)
         merged.defaults = self.defaults.merge(other.defaults)
-        merged.sections = self.sections.merge(other.sections)
+        merged.sections = {}
+        (self.sections.keys + other.sections.keys).uniq.each do |section_key|
+          self_section = self.sections[section_key]
+          other_section = other.sections[section_key]
+          merged_section = if self_section && other_section
+                             self_section.merge(other_section)
+                           elsif self_section || other_section
+                             self_section || other_section
+                           else
+                             raise "BUG: both of self and other section are nil"
+                           end
+          merged.sections[section_key] = merged_section
+        end
+
+        merged
+      end
+
+      def merge_for_finalized(other)
+        # list what subclass can do for finalized section
+        #  * overwrite param_name to escape duplicated name of instance variable
+        #  * append params/defaults/sections which are missing in superclass
+
+        options = {
+          param_name: other.param_name,
+          required: (self.required.nil? ? other.required : self.required),
+          multi: (self.multi.nil? ? other.multi : self.multi),
+          final: true,
+        }
+        merged = self.class.new(other.name, options)
+
+        merged.argument = self.argument || other.argument
+        merged.params = other.params.merge(self.params)
+        merged.defaults = other.defaults.merge(self.defaults)
+        merged.sections = {}
+        (self.sections.keys + other.sections.keys).uniq.each do |section_key|
+          self_section = self.sections[section_key]
+          other_section = other.sections[section_key]
+          merged_section = if self_section && other_section
+                             other_section.merge(self_section)
+                           elsif self_section || other_section
+                             self_section || other_section
+                           else
+                             raise "BUG: both of self and other section are nil"
+                           end
+          merged.sections[section_key] = merged_section
+        end
 
         merged
       end

data/lib/fluent/config/literal_parser.rb

@@ -147,7 +147,7 @@ module Fluent
 
         @ss.pos += code.length
 
-        code
+        '"#{' + code + '}"'
       end
 
       def eval_embedded_code(code)

data/lib/fluent/output.rb

@@ -98,7 +98,7 @@ module Fluent
     def initialize(output)
       @output = output
       @finish = false
-      @next_time = Engine.now + 1.0
+      @next_time = Time.now.to_f + 1.0
     end
 
     def configure(conf)
@@ -132,7 +132,7 @@ module Fluent
       @mutex.lock
       begin
         until @finish
-          time = Engine.now
+          time = Time.now.to_f
 
           if @next_time <= time
             @mutex.unlock
@@ -141,7 +141,7 @@ module Fluent
             ensure
               @mutex.lock
             end
-            next_wait = @next_time - Engine.now
+            next_wait = @next_time - Time.now.to_f
           else
             next_wait = @next_time - time
           end
@@ -231,7 +231,7 @@ module Fluent
     end
 
     def start
-      @next_flush_time = Engine.now + @flush_interval
+      @next_flush_time = Time.now.to_f + @flush_interval
       @buffer.start
       @secondary.start if @secondary
       @writers.each {|writer| writer.start }
@@ -280,10 +280,10 @@ module Fluent
     end
 
     def try_flush
-      time = Engine.now
+      time = Time.now.to_f
 
       empty = @buffer.queue_size == 0
-      if empty && @next_flush_time < (now = Engine.now)
+      if empty && @next_flush_time < (now = Time.now.to_f)
         @buffer.synchronize do
           if @next_flush_time < now
             enqueue_buffer
@@ -330,7 +330,7 @@ module Fluent
         end
 
         if has_next
-          return Engine.now + @queued_chunk_flush_interval
+          return Time.now.to_f + @queued_chunk_flush_interval
         else
           return time + @try_flush_interval
         end
@@ -385,7 +385,7 @@ module Fluent
 
     def force_flush
       @num_errors_lock.synchronize do
-        @next_retry_time = Engine.now - 1
+        @next_retry_time = Time.now.to_f - 1
       end
       enqueue_buffer(true)
       submit_flush

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -179,7 +179,7 @@ module Fluent
       end
 
       def expand(str)
-        str.gsub(/(\${[a-zA-Z0-9_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
+        str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
           log.warn "unknown placeholder `#{$1}` found" unless @placeholders.include?($1)
           @placeholders[$1]
         }

data/lib/fluent/plugin/out_forward.rb

@@ -298,10 +298,12 @@ module Fluent
               raw_data = sock.recv(1024)
 
               # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
-              # In this case, the node is available and successfully close connection without sending responses.
-              # ForwardInput is not expected to do so, but some alternatives may do so.
-              # Therefore do not send the chunk again.
-              unless raw_data.empty?
+              # If this happens we assume the data wasn't delivered and retry it.
+              if raw_data.empty?
+                @log.warn "node #{node.host}:#{node.port} closed the connection. regard it as unavailable."
+                node.disable!
+                raise ForwardOutputConnectionClosedError, "node #{node.host}:#{node.port} closed connection"
+              else
                 # Serialization type of the response is same as sent data.
                 res = MessagePack.unpack(raw_data)
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.7'
+  VERSION = '0.12.8'
 
 end

data/test/config/test_configurable.rb

@@ -100,14 +100,45 @@ module ConfigurableSpec
 
     config_param :name, :string, alias: :fullname
     config_param :bool, :bool, alias: :flag
-    config_section :detail, required: true, multi: false, alias: "information" do
-      config_param :address, :string
+    config_section :detail, required: false, multi: false, alias: "information" do
+      config_param :address, :string, default: "x"
     end
 
     def get_all
       [@name, @detail]
     end
   end
+
+  class Example2 < Example1
+    config_section :detail, required: true, multi: false, alias: "information2" do
+      config_param :address, :string, default: "y"
+      config_param :phone_no, :string
+    end
+  end
+
+  class Example3
+    include Fluent::Configurable
+
+    config_param :age, :integer, default: 10
+
+    config_section :appendix, required: true, multi: false, final: true do
+      config_param :type, :string
+      config_param :name, :string, default: "x"
+    end
+
+    def get_all
+      [@name, @detail]
+    end
+  end
+
+  class Example4 < Example3
+    config_param :age, :integer, default: 20
+
+    config_section :appendix, required: false, multi: false, final: false do
+      config_param :name, :string, default: "y"
+      config_param :age, :integer, default: 10
+    end
+  end
 end
 
 module Fluent::Config
@@ -493,6 +524,97 @@ module Fluent::Config
           assert_false(ex5.boolvalue)
         end
       end
+
+      sub_test_case '.config_section' do
+        def e(name, arg = '', attrs = {}, elements = [])
+          attrs_str_keys = {}
+          attrs.each{|key, value| attrs_str_keys[key.to_s] = value }
+          Fluent::Config::Element.new(name, arg, attrs_str_keys, elements)
+        end
+
+        test 'subclass configuration spec can overwrite superclass specs' do
+          # conf0 = e('ROOT', '', {}, [])
+
+          conf1 = e('ROOT', '', {
+              'name' => 'tagomoris',
+              'bool' => true,
+            },
+          )
+          # <detail> section is required by overwriting of Example2 config_section spec
+          assert_nothing_raised { ConfigurableSpec::Example1.new.configure(conf1) }
+          assert_raise(Fluent::ConfigError.new("'<detail>' sections are required")) { ConfigurableSpec::Example2.new.configure(conf1) }
+
+          conf2 = e('ROOT', '', {
+              'name' => 'tagomoris',
+              'bool' => true,
+            },
+            [e('detail', '', { 'phone_no' => "+81-00-0000-0000" }, [])],
+          )
+          # <detail> address </detail> default is overwritten by Example2
+          assert_nothing_raised { ConfigurableSpec::Example1.new.configure(conf2) }
+          assert_nothing_raised { ConfigurableSpec::Example2.new.configure(conf2) }
+          ex1 = ConfigurableSpec::Example1.new.configure(conf2)
+          assert_equal "x", ex1.detail.address
+          ex2 = ConfigurableSpec::Example2.new.configure(conf2)
+          assert_equal "y", ex2.detail.address
+
+          conf3 = e('ROOT', '', {
+              'name' => 'tagomoris',
+              'bool' => true,
+            },
+            [e('detail', '', { 'address' => "Chiyoda Tokyo Japan" }, [])],
+          )
+          # <detail> phone_no </detail> is required by Example2 config_param spec
+          assert_nothing_raised { ConfigurableSpec::Example1.new.configure(conf3) }
+          assert_raise(Fluent::ConfigError.new("'phone_no' parameter is required, in section detail")) { ConfigurableSpec::Example2.new.configure(conf3) }
+
+          conf4 = e('ROOT', '', {
+              'name' => 'tagomoris',
+              'bool' => true,
+            },
+            [e('detail', '', { 'address' => "Chiyoda Tokyo Japan", 'phone_no' => '+81-00-0000-0000' }, [])],
+          )
+          assert_nothing_raised { ConfigurableSpec::Example1.new.configure(conf4) } # phone_no is not used
+          assert_nothing_raised { ConfigurableSpec::Example2.new.configure(conf4) }
+
+          ex2 = ConfigurableSpec::Example2.new.configure(conf4)
+          assert_equal "Chiyoda Tokyo Japan", ex2.detail.address
+          assert_equal "+81-00-0000-0000", ex2.detail.phone_no
+        end
+
+        test 'adds only config_param definitions into configuration without overwriting existing finalized configuration elements' do
+
+          conf1 = e('ROOT', '', {}, [])
+          # <appendix> is required by Example3 and its not be overwritten by Example4
+          assert_raise(Fluent::ConfigError.new("'<appendix>' sections are required")) { ConfigurableSpec::Example3.new.configure(conf1) }
+          assert_raise(Fluent::ConfigError.new("'<appendix>' sections are required")) { ConfigurableSpec::Example4.new.configure(conf1) }
+
+          conf2 = e('ROOT', '', {
+            },
+            [e('appendix', '', {'type' => '1'}, [])],
+          )
+          # default value of age is overwritten by Example4, because root proxy is not finalized
+          ex3 = ConfigurableSpec::Example3.new.configure(conf2)
+          assert_equal 10, ex3.age
+          assert_equal '1', ex3.appendix.type
+          ex4 = ConfigurableSpec::Example4.new.configure(conf2)
+          assert_equal 20, ex4.age
+          assert_equal '1', ex4.appendix.type
+
+          conf3 = e('ROOT', '', {},
+            [e('appendix', '', {'type' => '2'}, [])],
+          )
+          # default value of <appendix> name </appendix> cannot be overwritten because it is finalized
+          ex3 = ConfigurableSpec::Example3.new.configure(conf2)
+          assert_equal 10, ex3.age
+          assert_equal '1', ex3.appendix.type
+          ex4 = ConfigurableSpec::Example4.new.configure(conf2)
+          assert_equal 20, ex4.age
+          assert_equal '1', ex4.appendix.type
+          # <appendix> age </appendix> can be added because it is missing in superclass spec
+          assert_equal 10, ex4.appendix.age
+        end
+      end
     end
 
     sub_test_case 'class defined with config_param/config_section having :alias' do

data/test/config/test_literal_parser.rb

@@ -228,6 +228,8 @@ module Fluent::Config
       test('"#{"}"}"') { assert_text_parsed_as("}", '"#{"}"}"') }
       test('"#{#}"') { assert_parse_error('"#{#}"') }  # error in embedded ruby code
       test("\"\#{\n=begin\n}\"") { assert_parse_error("\"\#{\n=begin\n}\"") }  # error in embedded ruby code
+      test('"#{v1}foo#{v2}"') { assert_text_parsed_as("#{v1}foo#{v2}", '"#{v1}foo#{v2}"') }
+      test('"#{1+1}foo#{2+2}bar"') { assert_text_parsed_as("#{1+1}foo#{2+2}bar", '"#{1+1}foo#{2+2}bar"') }
     end
 
     sub_test_case 'array parsing' do

data/test/plugin/test_filter_record_transformer.rb

@@ -343,5 +343,35 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         assert_nil(r['message'])
       end
     end
+
+    test 'expand fields starting with @ (enable_ruby no)' do
+      config = %[
+        enable_ruby no
+        <record>
+          foo ${@timestamp}
+        </record>
+      ]
+      d = create_driver(config)
+      message = {"@timestamp" => "foo"}
+      es = d.run { d.emit(message, @time) }.filtered
+      es.each do |t, r|
+        assert_equal(message["@timestamp"], r['foo'])
+      end
+    end
+
+    test 'expand fields starting with @ (enable_ruby yes)' do
+      config = %[
+        enable_ruby yes
+        <record>
+          foo ${__send__("@timestamp")}
+        </record>
+      ]
+      d = create_driver(config)
+      message = {"@timestamp" => "foo"}
+      es = d.run { d.emit(message, @time) }.filtered
+      es.each do |t, r|
+        assert_equal(message["@timestamp"], r['foo'])
+      end
+    end
   end
 end

data/test/plugin/test_out_forward.rb

@@ -186,6 +186,7 @@ class ForwardOutputTest < Test::Unit::TestCase
   end
 
   def test_require_a_node_not_supporting_responses_to_respond_with_ack
+    # in_forward, that doesn't support ack feature, and keep connection alive
     target_input_driver = create_target_input_driver
 
     d = create_driver(CONFIG + %[
@@ -203,6 +204,7 @@ class ForwardOutputTest < Test::Unit::TestCase
     d.register_run_post_condition do
       d.instance.responses.length == 1
     end
+    d.run_timeout = 2
 
     target_input_driver.run do
       d.run do
@@ -223,7 +225,48 @@ class ForwardOutputTest < Test::Unit::TestCase
     assert_equal 1, d.instance.exceptions.size
   end
 
-  def create_target_input_driver(do_respond=false, conf=TARGET_CONFIG)
+  # bdf1f4f104c00a791aa94dc20087fe2011e1fd83
+  def test_require_a_node_not_supporting_responses_2_to_respond_with_ack
+    # in_forward, that doesn't support ack feature, and disconnect immediately
+    target_input_driver = create_target_input_driver(false, true)
+
+    d = create_driver(CONFIG + %[
+      flush_interval 1s
+      require_ack_response true
+      ack_response_timeout 5s
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    d.register_run_post_condition do
+      d.instance.responses.length == 1
+    end
+    d.run_timeout = 2
+
+    target_input_driver.run do
+      d.run do
+        records.each do |record|
+          d.emit record, time
+        end
+      end
+    end
+
+    emits = target_input_driver.emits
+    assert_equal ['test', time, records[0]], emits[0]
+    assert_equal ['test', time, records[1]], emits[1]
+
+    assert_equal 0, d.instance.responses.size
+    assert_equal 1, d.instance.exceptions.size # send_data() fails and to be retried
+
+    node = d.instance.nodes.first
+    assert_equal false, node.available # node is regarded as unavailable when unexpected EOF
+  end
+
+  def create_target_input_driver(do_respond=false, disconnect=false, conf=TARGET_CONFIG)
     require 'fluent/plugin/in_forward'
 
     DummyEngineDriver.new(Fluent::ForwardInput) {
@@ -265,9 +308,13 @@ class ForwardOutputTest < Test::Unit::TestCase
                 # chunk_counter is reset to zero only after all the data have been received and successfully deserialized.
                 break if handler.chunk_counter == 0
               end
+              if disconnect
+                handler.close
+                sock = nil
+              end
               sleep  # wait for connection to be closed by client
             ensure
-              sock.close
+              sock.close if sock
             end
           end
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.7
+  version: 0.12.8
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-21 00:00:00.000000000 Z
+date: 2015-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack