fluentd 0.12.16 → 0.12.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2bc899dd92a360462c8ab40bdec4dfa955918c74
-  data.tar.gz: 0c93a1e695baf87d2d01c0ff10821a2733115c44
+  metadata.gz: 9a8a90a863a94d939296d18ce47d4a37d4568ae6
+  data.tar.gz: 4ccca67d8a418215bca28e7b2297292b6782ca7d
 SHA512:
-  metadata.gz: 35585caea38a587df230faa0027cbd5820c6e43e48669d35755055f73a185509e08bf0089c0ff7b2e4c4eaacfe361406f14709161d81f7e3ebc6354c5a203fd1
-  data.tar.gz: 9f3e64fedf79a5609c54e7387a14d2a4c03705b7be00d4382d554a4ff357b5617dcc10689f6e4067ecd8e9e99cf6158583c88ac32761433f04fb091229db6fdc
+  metadata.gz: 95398c478a7b63b0b2f497cd4cdda0507ca9af63b6f55d9f9c76f6efe9c377c08e05864e67d9049cddb787082bc911ca04990ad2044a810ef9ec556de0fcd876
+  data.tar.gz: 58ad404181a58040ede0b7e1fd8bba4311ddae24378b3f777708092bdfe3a32d74b836602c1e003a0eb1a9ef6a98f3e0836a274ced77ccf2cf99da7adc3e62f3

data/.travis.yml

@@ -16,6 +16,7 @@ branches:
   only:
     - master
     - v0.10
+    - v0.12
     - v0.14
 
 gemfile:

data/ChangeLog

@@ -1,5 +1,26 @@
 # v0.12
 
+## Release 0.12.17 - 2015/11/04
+
+### New features / Enhancement
+
+* Engine: Add Engine#msgpack_factory for v0.14 compatibility
+  https://github.com/fluent/fluentd/pull/693
+* Log shutdown plugin info to check shutdown sequence
+  https://github.com/fluent/fluentd/pull/689
+* in_monitor_agent: Emit plugin info to use existing plugins
+  https://github.com/fluent/fluentd/pull/670
+* config: Improve describing plugin parameters
+  https://github.com/fluent/fluentd/pull/683
+
+### Bug fixes
+
+* Tempfile should be binary mode
+  https://github.com/fluent/fluentd/pull/691
+* filter_record_transformer: Don't use popular name to prevent field overwrite when enable_ruby is true
+  https://github.com/fluent/fluentd/pull/687
+
+
 ## Release 0.12.16 - 2015/09/30
 
 ### New features / Enhancement

data/README.md

@@ -38,6 +38,7 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
 - Documentation: http://docs.fluentd.org/
 - Source repository: http://github.com/fluent
 - Discussion: http://groups.google.com/group/fluentd
+- Slack / Community: http://slack.fluentd.org
 - Newsletters: http://get.treasuredata.com/Fluentd_education
 - Author: Sadayuki Furuhashi
 - Copyright: (c) 2011 FURUHASHI Sadayuki
@@ -48,4 +49,3 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).
 
 [<img src="https://ga-beacon.appspot.com/UA-24890265-6/fluent/fluentd" />](https://github.com/fluent/fluentd)
-

data/fluentd.gemspec

@@ -35,6 +35,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("simplecov", ["~> 0.6.4"])
   gem.add_development_dependency("rr", [">= 1.0.0"])
   gem.add_development_dependency("timecop", [">= 0.3.0"])
-  gem.add_development_dependency("test-unit", ["~> 3.0.2"])
+  gem.add_development_dependency("test-unit", ["~> 3.1.4"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0.3"])
 end

data/lib/fluent/agent.rb

@@ -78,6 +78,7 @@ module Fluent
       @started_filters.map { |f|
         Thread.new do
           begin
+            log.info "shutting down filter#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(f.class), plugin_id: f.plugin_id
             f.shutdown
           rescue => e
             log.warn "unexpected error while shutting down filter plugins", :plugin => f.class, :plugin_id => f.plugin_id, :error_class => e.class, :error => e
@@ -91,6 +92,7 @@ module Fluent
       @started_outputs.map { |o|
         Thread.new do
           begin
+            log.info "shutting down output#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(o.class), plugin_id: o.plugin_id
             o.shutdown
           rescue => e
             log.warn "unexpected error while shutting down output plugins", :plugin => o.class, :plugin_id => o.plugin_id, :error_class => e.class, :error => e

data/lib/fluent/buffer.rb

@@ -112,7 +112,7 @@ module Fluent
 
     def msgpack_each(&block)
       open {|io|
-        u = MessagePack::Unpacker.new(io)
+        u = Fluent::Engine.msgpack_factory.unpacker(io)
         begin
           u.each(&block)
         rescue EOFError

data/lib/fluent/command/cat.rb

@@ -289,7 +289,7 @@ when 'json'
 
 when 'msgpack'
   begin
-    u = MessagePack::Unpacker.new($stdin)
+    u = Fluent::Engine.msgpack_factory.unpacker($stdin)
     u.each {|record|
       w.write(record)
     }

data/lib/fluent/config/configure_proxy.rb

@@ -178,6 +178,11 @@ module Fluent
       def config_param(name, *args, &block)
         name, block, opts = parameter_configuration(name, *args, &block)
 
+        if @current_description
+          config_set_desc(name, @current_description)
+          @current_description = nil
+        end
+
         @sections.delete(name)
         @params[name] = [block, opts]
         name
@@ -205,6 +210,10 @@ module Fluent
         nil
       end
 
+      def desc(description)
+        @current_description = description
+      end
+
       def config_section(name, *args, &block)
         unless block_given?
           raise ArgumentError, "#{self.name}: config_section requires block parameter"

data/lib/fluent/configurable.rb

@@ -105,11 +105,19 @@ module Fluent
         configure_proxy(self.name).config_set_default(name, defval)
       end
 
+      def config_set_desc(name, desc)
+        configure_proxy(self.name).config_set_desc(name, desc)
+      end
+
       def config_section(name, *args, &block)
         configure_proxy(self.name).config_section(name, *args, &block)
         attr_accessor configure_proxy(self.name).sections[name].param_name
       end
 
+      def desc(description)
+        configure_proxy(self.name).desc(description)
+      end
+
       def merged_configure_proxy
         configurables = ancestors.reverse.select{ |a| a.respond_to?(:configure_proxy) }
 

data/lib/fluent/engine.rb

@@ -19,6 +19,16 @@ module Fluent
   require 'fluent/root_agent'
 
   class EngineClass
+    class DummyMessagePackFactory
+      def packer(*args)
+        MessagePack::Packer.new(*args)
+      end
+
+      def unpacker(*args)
+        MessagePack::Unpacker.new(*args)
+      end
+    end
+
     def initialize
       @root_agent = nil
       @event_router = nil
@@ -30,6 +40,8 @@ module Fluent
       @log_event_queue = []
 
       @suppress_config_dump = false
+
+      @msgpack_factory = DummyMessagePackFactory.new
     end
 
     MATCH_CACHE_SIZE = 1024
@@ -37,6 +49,7 @@ module Fluent
 
     attr_reader :root_agent
     attr_reader :matches, :sources
+    attr_reader :msgpack_factory
 
     def init(opts = {})
       BasicSocket.do_not_reverse_lookup = true

data/lib/fluent/event.rb

@@ -27,7 +27,7 @@ module Fluent
     end
 
     def to_msgpack_stream
-      out = MessagePack::Packer.new # MessagePack::Packer is fastest way to serialize events
+      out = Fluent::Engine.msgpack_factory.packer
       each {|time,record|
         out.write([time,record])
       }
@@ -143,7 +143,7 @@ module Fluent
 
     def each(&block)
       # TODO format check
-      unpacker = MessagePack::Unpacker.new
+      unpacker = Fluent::Engine.msgpack_factory.unpacker
       unpacker.feed_each(@data, &block)
       nil
     end

data/lib/fluent/plugin/buf_memory.rb

@@ -57,7 +57,7 @@ module Fluent
 
     # optimize
     def msgpack_each(&block)
-      u = MessagePack::Unpacker.new
+      u = Fluent::Engine.msgpack_factory.unpacker
       u.feed_each(@data, &block)
     end
   end

data/lib/fluent/plugin/exec_util.rb

@@ -58,7 +58,7 @@ module Fluent
 
     class MessagePackParser < Parser
       def call(io)
-        @u = MessagePack::Unpacker.new(io)
+        @u = Fluent::Engine.msgpack_factory.unpacker(io)
         begin
           @u.each(&@on_message)
         rescue EOFError

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -230,18 +230,17 @@ module Fluent
         @placeholders = struct
       end
 
-      def expand(str, force_stringify=false)
+      def expand(_str_for_eval_, force_stringify=false)
         if @auto_typecast and !force_stringify
-          single_placeholder_matched = str.match(/\A\${([^}]+)}\z/)
-          if single_placeholder_matched
-            code = single_placeholder_matched[1]
-            return eval code, @placeholders.instance_eval { binding }
+          _single_placeholder_matched_ = _str_for_eval_.match(/\A\${([^}]+)}\z/)
+          if _single_placeholder_matched_
+            return eval _single_placeholder_matched_[1], @placeholders.instance_eval { binding }
           end
         end
-        interpolated = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
-        eval "\"#{interpolated}\"", @placeholders.instance_eval { binding }
+        _interpolated_for_eval_ = _str_for_eval_.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
+        eval "\"#{_interpolated_for_eval_}\"", @placeholders.instance_eval { binding }
       rescue => e
-        log.warn "failed to expand `#{str}`", :error_class => e.class, :error => e.message
+        log.warn "failed to expand `#{_str_for_eval_}`", :error_class => e.class, :error => e.message
         log.warn_backtrace
         nil
       end

data/lib/fluent/plugin/in_dummy.rb

@@ -20,9 +20,13 @@ module Fluent
 
     BIN_NUM = 10
 
+    desc "The value is the tag assigned to the generated events."
     config_param :tag, :string
+    desc "It configures how many events to generate per second."
     config_param :rate, :integer, :default => 1
+    desc "If specified, each generated event has an auto-incremented key field."
     config_param :auto_increment_key, :string, :default => nil
+    desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
     config_param :dummy, :default => [{"message"=>"dummy"}] do |val|
       begin
         parsed = JSON.parse(val)

data/lib/fluent/plugin/in_forward.rb

@@ -219,7 +219,7 @@ module Fluent
         else
           m = method(:on_read_msgpack)
           @serializer = :to_msgpack.to_proc
-          @u = MessagePack::Unpacker.new
+          @u = Fluent::Engine.msgpack_factory.unpacker
         end
 
         (class << self; self; end).module_eval do

data/lib/fluent/plugin/in_http.rb

@@ -191,7 +191,7 @@ module Fluent
 
     def parse_params_default(params)
       record = if msgpack = params['msgpack']
-                 MessagePack.unpack(msgpack)
+                 Engine.msgpack_factory.unpacker.feed(msgpack).read
                elsif js = params['json']
                  JSON.parse(js)
                else

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -27,6 +27,8 @@ module Fluent
 
     config_param :bind, :string, :default => '0.0.0.0'
     config_param :port, :integer, :default => 24220
+    config_param :tag, :string, :default => nil
+    config_param :emit_interval, :time, :default => 60
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -199,6 +201,36 @@ module Fluent
       end
     end
 
+    class TimerWatcher < Coolio::TimerWatcher
+      def initialize(interval, log, &callback)
+        @callback = callback
+        @log = log
+
+        # Avoid long shutdown time
+        @num_call = 0
+        if interval >= 10
+          min_interval = 10
+          @call_interval = interval / 10
+        else
+          min_interval = interval
+          @call_interval = 0
+        end
+
+        super(min_interval, true)
+      end
+
+      def on_timer
+        @num_call += 1
+        if @num_call >= @call_interval
+          @num_call = 0
+          @callback.call
+        end
+      rescue => e
+        @log.error e.to_s
+        @log.error_backtrace
+      end
+    end
+
     def start
       log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
       @srv = WEBrick::HTTPServer.new({
@@ -214,6 +246,29 @@ module Fluent
       @thread = Thread.new {
         @srv.start
       }
+      if @tag
+        log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
+
+        @loop = Coolio::Loop.new
+        opts = {:with_config => false}
+        timer = TimerWatcher.new(@emit_interval, log) {
+          es = MultiEventStream.new
+          now = Engine.now
+          plugins_info_all(opts).each { |record|
+            es.add(now, record)
+          }
+          router.emit_stream(@tag, es)
+        }
+        @loop.attach(timer)
+        @thread_for_emit = Thread.new(&method(:run))
+      end
+    end
+
+    def run
+      @loop.run
+    rescue => e
+      log.error "unexpected error", :error => e.to_s
+      log.error_backtrace
     end
 
     def shutdown
@@ -225,6 +280,13 @@ module Fluent
         @thread.join
         @thread = nil
       end
+      if @tag
+        @loop.watchers.each { |w| w.detach }
+        @loop.stop
+        @loop = nil
+        @thread_for_emit.join
+        @thread_for_emit = nil
+      end
     end
 
     MONITOR_INFO = {
@@ -320,7 +382,7 @@ module Fluent
       obj['plugin_id'] = pe.plugin_id
       obj['plugin_category'] = plugin_category(pe)
       obj['type'] = pe.config['@type'] || pe.config['type']
-      obj['config'] = pe.config
+      obj['config'] = pe.config if !opts.has_key?(:with_config) || opts[:with_config]
 
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|

data/lib/fluent/plugin/in_stream.rb

@@ -130,7 +130,7 @@ module Fluent
           @y.on_parse_complete = @on_message
         else
           m = method(:on_read_msgpack)
-          @u = MessagePack::Unpacker.new
+          @u = Fluent::Engine.msgpack_factory.unpacker
         end
 
         (class << self; self; end).module_eval do

data/lib/fluent/plugin/out_exec.rb

@@ -81,6 +81,7 @@ module Fluent
         prog = "#{@command} #{chunk.path}"
       else
         tmpfile = Tempfile.new("fluent-plugin-exec-")
+        tmpfile.binmode
         chunk.write_to(tmpfile)
         tmpfile.close
         prog = "#{@command} #{tmpfile.path}"

data/lib/fluent/plugin/out_stream.rb

@@ -65,7 +65,7 @@ module Fluent
         chain = NullOutputChain.instance
         chunk.open {|io|
           # TODO use MessagePackIoEventStream
-          u = MessagePack::Unpacker.new(io)
+          u = Fluent::Engine.msgpack_factory.unpacker(io)
           begin
             u.each {|(tag,entries)|
               es = MultiEventStream.new

data/lib/fluent/process.rb

@@ -176,7 +176,7 @@ module Fluent
     end
 
     def read_event_stream(r, &block)
-      u = MessagePack::Unpacker.new(r)
+      u = Fluent::Engine.msgpack_factory.unpacker(r)
       begin
         #buf = ''
         #map = {}

data/lib/fluent/root_agent.rb

@@ -118,6 +118,7 @@ module Fluent
       @started_inputs.map { |i|
         Thread.new do
           begin
+            log.info "shutting down input", type: Plugin.lookup_name_from_class(i.class), plugin_id: i.plugin_id
             i.shutdown
           rescue => e
             log.warn "unexpected error while shutting down input plugin", :plugin => i.class, :plugin_id => i.plugin_id, :error_class => e.class, :error => e

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.16'
+  VERSION = '0.12.17'
 
 end

data/test/config/test_configure_proxy.rb

@@ -112,6 +112,26 @@ module Fluent::Config
         end
       end
 
+      sub_test_case '#desc' do
+        setup do
+          @proxy = Fluent::Config::ConfigureProxy.new(:section)
+        end
+
+        test 'permit to specify description twice' do
+          @proxy.desc("description1")
+          @proxy.desc("description2")
+          @proxy.config_param(:name, :string)
+          assert_equal("description2", @proxy.descriptions[:name])
+        end
+
+        test 'does not permit description specification twice' do
+          @proxy.desc("description1")
+          assert_raise(ArgumentError) do
+            @proxy.config_param(:name, :string, :desc => "description2")
+          end
+        end
+      end
+
       sub_test_case '#dump' do
         setup do
           @proxy = Fluent::Config::ConfigureProxy.new(:section)
@@ -206,6 +226,28 @@ CONFIG
             end
             assert_equal(<<CONFIG, @proxy.dump)
 
+sub
+ name1: string: <"name1"> # desc1
+ name2: string: <"name2"> # desc2
+ sub2
+  name3: string: <"name3">
+  name4: string: <"name4"> # desc4
+CONFIG
+          end
+
+          test 'sub proxy w/ desc method' do
+            @proxy.config_section(:sub) do
+              desc("desc1")
+              config_param(:name1, :string, default: "name1")
+              config_param(:name2, :string, default: "name2", desc: "desc2")
+              config_section(:sub2) do
+                config_param(:name3, :string, default: "name3")
+                desc("desc4")
+                config_param(:name4, :string, default: "name4")
+              end
+            end
+            assert_equal(<<CONFIG, @proxy.dump)
+
 sub
  name1: string: <"name1"> # desc1
  name2: string: <"name2"> # desc2

data/test/plugin/test_in_forward.rb

@@ -59,7 +59,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       d.expected_emits.each {|tag,time,record|
-        send_data [tag, time, record].to_msgpack
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
       }
     end
   end
@@ -77,7 +77,7 @@ class ForwardInputTest < Test::Unit::TestCase
       d.expected_emits.each {|tag,time,record|
         entries << [time, record]
       }
-      send_data ["tag1", entries].to_msgpack
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
   end
 
@@ -92,9 +92,9 @@ class ForwardInputTest < Test::Unit::TestCase
     d.run do
       entries = ''
       d.expected_emits.each {|tag,time,record|
-        [time, record].to_msgpack(entries)
+        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
       }
-      send_data ["tag1", entries].to_msgpack
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
   end
 
@@ -128,7 +128,7 @@ class ForwardInputTest < Test::Unit::TestCase
     assert chunk.size < (32 * 1024 * 1024)
 
     d.run do
-      MessagePack::Unpacker.new.feed_each(chunk) do |obj|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
         d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
       end
     end
@@ -157,7 +157,7 @@ class ForwardInputTest < Test::Unit::TestCase
     chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
 
     d.run do
-      MessagePack::Unpacker.new.feed_each(chunk) do |obj|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
         d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
       end
     end
@@ -184,7 +184,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     # d.run => send_data
     d.run do
-      MessagePack::Unpacker.new.feed_each(chunk) do |obj|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
         d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
       end
     end

data/test/plugin/test_in_stream.rb

@@ -17,7 +17,7 @@ module StreamInputTest
 
     d.run do
       d.expected_emits.each {|tag,time,record|
-        send_data [tag, 0, record].to_msgpack
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, 0, record]).to_s
       }
     end
   end
@@ -32,7 +32,7 @@ module StreamInputTest
 
     d.run do
       d.expected_emits.each {|tag,time,record|
-        send_data [tag, time, record].to_msgpack
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
       }
     end
   end
@@ -50,7 +50,7 @@ module StreamInputTest
       d.expected_emits.each {|tag,time,record|
         entries << [time, record]
       }
-      send_data ["tag1", entries].to_msgpack
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
   end
 
@@ -65,9 +65,9 @@ module StreamInputTest
     d.run do
       entries = ''
       d.expected_emits.each {|tag,time,record|
-        [time, record].to_msgpack(entries)
+        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
       }
-      send_data ["tag1", entries].to_msgpack
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
   end
 

data/test/plugin/test_out_copy.rb

@@ -86,7 +86,7 @@ class CopyOutputTest < Test::Unit::TestCase
 
     es = if defined?(MessagePack::Packer)
            time = Time.parse("2013-05-26 06:37:22 UTC").to_i
-           packer = MessagePack::Packer.new
+           packer = Fluent::Engine.msgpack_factory.packer
            packer.pack([time, {"a" => 1}])
            packer.pack([time, {"a" => 2}])
            Fluent::MessagePackEventStream.new(packer.to_s)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.16
+  version: 0.12.17
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-30 00:00:00.000000000 Z
+date: 2015-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -244,14 +244,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.2
+        version: 3.1.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.2
+        version: 3.1.4
 - !ruby/object:Gem::Dependency
   name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
@@ -465,7 +465,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.3
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector