fluentd 0.10.46 → 0.10.47

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5c5c7236589cb3aa5fadf040dbee965523f6ef0
-  data.tar.gz: 392f590233abe73edab6c425556de56ab7ddd6f3
+  metadata.gz: 2fe2bff2ce1a5aecb6cee52438d601388c2aa896
+  data.tar.gz: f8aed6a81a228824681acde0c7f38656c4ea17b8
 SHA512:
-  metadata.gz: ea036af28812b5a21fafe9731fe93959f85c2529e88feff2347d1ebfb0724c54a93a65bd738371cc924247e43ef9c6f997dd88792893fbb94c2fbdfc52fa362d
-  data.tar.gz: 6087c6298709c8f0e32f79830c8950ab5378225c8a68ecf8ae236accdf2e08b670e51dbd7c08ac2e4397a36d704e4219a4585ceebb67ab974428cdce578e806d
+  metadata.gz: 59337e23670288e3d11cbbf954d2601cd79d6f32b9cb008ffd94c3b59f5e1a5272774be6640d0a7c2af2fef26cca690e88422752d35f9b64beae2383627eb297
+  data.tar.gz: 1ed344c08218d378a029dc575083fd1217b1c227a38030e9907eca5b8d445e534bc04e7e068c0a79c8cb4324cc015568174e761ef3dcc3b347db1e91a239cf1b

data/.travis.yml

@@ -3,11 +3,14 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1.0
-  - 2.1.1
+  - 2.1
   - ruby-head
   - rbx
 
+os:
+  - linux
+  - osx
+
 branches:
   only:
     - master

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 0.10.47 - 2014/05/15
+
+* in_tail: Fix typo in flush_buffer
+* in_forward: Add chunk_size_warn_limit and chunk_size_limit for large chunks
+* in_monitor_agent: Add /api/config APIs to expose Fluentd configurations
+* supervisor: Add Supervisor.default_options to return default Fluentd options
+* test: Add DummyLogDevise and improve log handling in tests
+* config: Allow empty string value like `tag_mapped` in v1 configuration
+
 Release 0.10.46 - 2014/04/21
 
 * in_tail: Fix typo in rescue for dumping backtrace

data/lib/fluent/command/fluentd.rb

@@ -19,26 +19,13 @@
 require 'optparse'
 require 'fluent/log'
 require 'fluent/env'
+require 'fluent/supervisor'
 require 'fluent/version'
 
 op = OptionParser.new
 op.version = Fluent::VERSION
 
-# default values
-opts = {
-  :config_path => Fluent::DEFAULT_CONFIG_PATH,
-  :plugin_dirs => [Fluent::DEFAULT_PLUGIN_DIR],
-  :log_level => Fluent::Log::LEVEL_INFO,
-  :log_path => nil,
-  :daemonize => false,
-  :libs => [],
-  :setup_path => nil,
-  :chuser => nil,
-  :chgroup => nil,
-  :suppress_interval => 0,
-  :suppress_repeated_stacktrace => false,
-  :use_v1_config => false,
-}
+opts = Fluent::Supervisor.default_options
 
 op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "install sample configuration file to the directory") {|s|
   opts[:setup_path] = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)

data/lib/fluent/config/basic_parser.rb

@@ -65,6 +65,10 @@ module Fluent
         @ss.eos?
       end
 
+      def prev_match
+        @ss[0]
+      end
+
       def line_end
         skip(LINE_END)
       end

data/lib/fluent/config/types.rb

@@ -38,6 +38,8 @@ module Fluent
         true
       when 'false', 'no'
         false
+      when ''
+        true
       else
         nil
       end

data/lib/fluent/config/v1_parser.rb

@@ -102,11 +102,15 @@ module Fluent
           else
             k = scan_string(SPACING)
             spacing
-            v = parse_literal
-            unless line_end
-              parse_error! "expected end of line"
+            if prev_match.include?("\n") # support 'tag_mapped' like "without value" configuration
+              attrs[k] = ""
+            else
+              v = parse_literal
+              unless line_end
+                parse_error! "expected end of line"
+              end
+              attrs[k] = v
             end
-            attrs[k] = v
           end
         end
 

data/lib/fluent/log.rb

@@ -357,7 +357,9 @@ module Fluent
       super
 
       if @log_level
-        @log = PluginLogger.new($log)
+        unless @log.is_a?(PluginLogger)
+          @log = PluginLogger.new($log)
+        end
         @log.level = @log_level
       end
     end

data/lib/fluent/plugin/in_forward.rb

@@ -34,6 +34,9 @@ module Fluent
     # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
     config_param :blocking_timeout, :time, :default => 0.5
 
+    config_param :chunk_size_warn_limit, :size, :default => nil
+    config_param :chunk_size_limit, :size, :default => nil
+
     def configure(conf)
       super
     end
@@ -122,7 +125,7 @@ module Fluent
     #   2: long? time
     #   3: object record
     # }
-    def on_message(msg)
+    def on_message(msg, chunk_size, source)
       if msg.nil?
         # for future TCP heartbeat_request
         return
@@ -132,6 +135,13 @@ module Fluent
       tag = msg[0].to_s
       entries = msg[1]
 
+      if @chunk_size_limit && (chunk_size > @chunk_size_limit)
+        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source, limit: @chunk_size_limit, size: chunk_size
+        return
+      elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
+        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source, limit: @chunk_size_warn_limit, size: chunk_size
+      end
+
       if entries.class == String
         # PackedForward
         es = MessagePackEventStream.new(entries, @cached_unpacker)
@@ -160,16 +170,29 @@ module Fluent
     end
 
     class Handler < Coolio::Socket
+      PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
+
       def initialize(io, linger_timeout, log, on_message)
         super(io)
-        if io.is_a?(TCPSocket)
+
+        if io.is_a?(TCPSocket) # for unix domain socket support in the future
+          proto, port, host, addr = ( io.peeraddr rescue PEERADDR_FAILED )
+          @source = "host: #{host}, addr: #{addr}, port: #{port}"
+
           opt = [1, linger_timeout].pack('I!I!')  # { int l_onoff; int l_linger; }
           io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
         end
+
+        @chunk_counter = 0
         @on_message = on_message
         @log = log
         @log.trace {
-          remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername) rescue nil
+          begin
+            remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername)
+          rescue => e
+            remote_port = nil
+            remote_addr = nil
+          end
           "accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
         }
       end
@@ -182,7 +205,10 @@ module Fluent
         if first == '{' || first == '['
           m = method(:on_read_json)
           @y = Yajl::Parser.new
-          @y.on_parse_complete = @on_message
+          @y.on_parse_complete = lambda { |obj|
+            @on_message.call(obj, @chunk_counter, @source)
+            @chunk_counter = 0
+          }
         else
           m = method(:on_read_msgpack)
           @u = MessagePack::Unpacker.new
@@ -195,6 +221,7 @@ module Fluent
       end
 
       def on_read_json(data)
+        @chunk_counter += data.bytesize
         @y << data
       rescue => e
         @log.error "forward error", :error => e, :error_class => e.class
@@ -203,7 +230,11 @@ module Fluent
       end
 
       def on_read_msgpack(data)
-        @u.feed_each(data, &@on_message)
+        @chunk_counter += data.bytesize
+        @u.feed_each(data) do |obj|
+          @on_message.call(obj, @chunk_counter, @source)
+          @chunk_counter = 0
+        end
       rescue => e
         @log.error "forward error", :error => e, :error_class => e.class
         @log.error_backtrace

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -153,6 +153,36 @@ module Fluent
       end
     end
 
+    class ConfigMonitorServlet < MonitorServlet
+      def build_object(req, res)
+        {
+          'pid' => Process.pid,
+          'ppid' => Process.ppid
+        }.merge(@agent.fluentd_opts)
+      end
+    end
+
+    class LTSVConfigMonitorServlet < ConfigMonitorServlet
+      def process(req, res)
+        result = build_object(req, res)
+
+        row = []
+        JSON.parse(result.to_json).each_pair { |k, v|
+          row << "#{k}:#{v}"
+        }
+        text = row.join("\t")
+
+        [200, {'Content-Type'=>'text/plain'}, text]
+      end
+    end
+
+    class JSONConfigMonitorServlet < ConfigMonitorServlet
+      def process(req, res)
+        result = build_object(req, res)
+        render_json(result)
+      end
+    end
+
     def start
       log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
       @srv = WEBrick::HTTPServer.new({
@@ -163,6 +193,8 @@ module Fluent
         })
       @srv.mount('/api/plugins', LTSVMonitorServlet, self)
       @srv.mount('/api/plugins.json', JSONMonitorServlet, self)
+      @srv.mount('/api/config', LTSVConfigMonitorServlet, self)
+      @srv.mount('/api/config.json', JSONConfigMonitorServlet, self)
       @thread = Thread.new {
         @srv.start
       }
@@ -282,5 +314,18 @@ module Fluent
 
       obj
     end
+
+    def fluentd_opts
+      @fluentd_opts ||= get_fluentd_opts
+    end
+
+    def get_fluentd_opts
+      opts = {}
+      ObjectSpace.each_object(Fluent::Supervisor) { |obj|
+        opts.merge!(obj.options)
+        break
+      }
+      opts
+    end
   end
 end

data/lib/fluent/plugin/in_tail.rb

@@ -188,7 +188,7 @@ module Fluent
         time, record = parse_line(lb)
         if time && record
           tag = if @tag_prefix || @tag_suffix
-                  @tag_prefix + tail_watcher.tag + @tag_suffix
+                  @tag_prefix + tw.tag + @tag_suffix
                 else
                   @tag
                 end

data/lib/fluent/supervisor.rb

@@ -15,6 +15,10 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
+require 'fluent/env'
+require 'fluent/log'
+
 module Fluent
   class Supervisor
     class LoggerInitializer
@@ -56,6 +60,23 @@ module Fluent
       end
     end
 
+    def self.default_options
+      {
+        :config_path => Fluent::DEFAULT_CONFIG_PATH,
+        :plugin_dirs => [Fluent::DEFAULT_PLUGIN_DIR],
+        :log_level => Fluent::Log::LEVEL_INFO,
+        :log_path => nil,
+        :daemonize => nil,
+        :libs => [],
+        :setup_path => nil,
+        :chuser => nil,
+        :chgroup => nil,
+        :suppress_interval => 0,
+        :suppress_repeated_stacktrace => false,
+        :use_v1_config => false,
+      }
+    end
+
     def initialize(opt)
       @config_path = opt[:config_path]
       @log_path = opt[:log_path]
@@ -99,6 +120,15 @@ module Fluent
       end
     end
 
+    def options
+      {
+        'config_path' => @config_path,
+        'pid_file' => @daemonize,
+        'plugin_dirs' => @plugin_dirs,
+        'log_path' => @log_path
+      }
+    end
+
     private
 
     def dry_run
@@ -388,4 +418,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/test.rb

@@ -4,5 +4,3 @@ require 'fluent/test/base'
 require 'fluent/test/input_test'
 require 'fluent/test/output_test'
 
-$log ||= Fluent::Log.new
-

data/lib/fluent/test/base.rb

@@ -32,6 +32,8 @@ module Fluent
         else
           @instance = klass
         end
+        @instance.log = TestLogger.new
+
         @config = Config.new
       end
 
@@ -58,5 +60,44 @@ module Fluent
         end
       end
     end
+
+    class DummyLogDevice
+      attr_reader :logs
+
+      def initialize
+        @logs = []
+      end
+
+      def tty?
+        false
+      end
+
+      def puts(*args)
+        args.each{ |arg| write(arg + "\n") }
+      end
+
+      def write(message)
+        @logs.push message
+      end
+
+      def flush
+        true
+      end
+
+      def close
+        true
+      end
+    end
+
+    class TestLogger < Fluent::PluginLogger
+      def initialize
+        @logdev = DummyLogDevice.new
+        super(Fluent::Log.new(@logdev))
+      end
+
+      def logs
+        @logdev.logs
+      end
+    end
   end
 end

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.46'
+  VERSION = '0.10.47'
 
 end

data/spec/config/config_parser_spec.rb

@@ -34,6 +34,13 @@ describe Fluent::Config::V1Parser do
       ].should be_parsed_as("k1"=>"v1", "k2"=>"v2")
     end
 
+    it "allows attribute without value" do
+      %[
+        k1
+        k2 v2
+      ].should be_parsed_as("k1"=>"", "k2"=>"v2")
+    end
+
     it "parses attribute key always string" do
       "1 1".should be_parsed_as("1" => "1")
     end

data/test/helper.rb

@@ -1,6 +1,7 @@
 require 'test/unit'
 require 'fileutils'
 require 'fluent/log'
+require 'fluent/test'
 require 'rr'
 
 unless defined?(Test::Unit::AssertionFailedError)
@@ -26,4 +27,4 @@ def ipv6_enabled?
   end
 end
 
-$log = Fluent::Log.new(STDOUT, Fluent::Log::LEVEL_WARN)
+$log = Fluent::Log.new(Fluent::Test::DummyLogDevice.new, Fluent::Log::LEVEL_WARN)

data/test/plugin/test_in_forward.rb

@@ -116,6 +116,93 @@ class ForwardInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_send_large_chunk_warning
+    d = create_driver(CONFIG + %[
+      chunk_size_warn_limit 16M
+      chunk_size_limit 32M
+    ])
+
+    time = Time.parse("2014-04-25 13:14:15 UTC").to_i
+
+    # generate over 16M chunk
+    str = "X" * 1024 * 1024
+    chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
+    assert chunk.size > (16 * 1024 * 1024)
+    assert chunk.size < (32 * 1024 * 1024)
+
+    d.run do
+      MessagePack::Unpacker.new.feed_each(chunk) do |obj|
+        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+      end
+    end
+
+    # check emitted data
+    emits = d.emits
+    assert_equal 16, emits.size
+    assert emits.map(&:first).all?{|t| t == "test.tag" }
+    assert_equal (0...16).to_a, emits.map{|tag, t, record| t - time }
+
+    # check log
+    assert d.instance.log.logs.select{|line|
+      line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
+      line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
+    }.size == 1, "large chunk warning is not logged"
+  end
+
+  def test_send_large_chunk_only_warning
+    d = create_driver(CONFIG + %[
+      chunk_size_warn_limit 16M
+    ])
+    time = Time.parse("2014-04-25 13:14:15 UTC").to_i
+
+    # generate over 16M chunk
+    str = "X" * 1024 * 1024
+    chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
+
+    d.run do
+      MessagePack::Unpacker.new.feed_each(chunk) do |obj|
+        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+      end
+    end
+
+    # check log
+    assert d.instance.log.logs.select{ |line|
+      line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
+      line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
+    }.size == 1, "large chunk warning is not logged"
+  end
+
+  def test_send_large_chunk_limit
+    d = create_driver(CONFIG + %[
+      chunk_size_warn_limit 16M
+      chunk_size_limit 32M
+    ])
+
+    time = Time.parse("2014-04-25 13:14:15 UTC").to_i
+
+    # generate over 32M chunk
+    str = "X" * 1024 * 1024
+    chunk = [ "test.tag", (0...32).map{|i| [time + i, {"data" => str}] } ].to_msgpack
+    assert chunk.size > (32 * 1024 * 1024)
+
+    # d.run => send_data
+    d.run do
+      MessagePack::Unpacker.new.feed_each(chunk) do |obj|
+        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+      end
+    end
+
+    # check emitted data
+    emits = d.emits
+    assert_equal 0, emits.size
+
+    # check log
+    assert d.instance.log.logs.select{|line|
+      line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
+      line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=33554432 size=33554989/
+    }.size == 1, "large chunk warning is not logged"
+  end
+
   def send_data(data)
     io = connect
     begin

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.46
+  version: 0.10.47
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-21 00:00:00.000000000 Z
+date: 2014-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -253,7 +253,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".rvmrc"
 - ".travis.yml"
 - AUTHORS
 - COPYING

data/.rvmrc

@@ -1 +0,0 @@
-rvm 1.9.3