fluentd 0.10.41 → 0.10.42

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1f0356ea605c03d2a1bdac6b5d0c391efbc7d1d0
+  data.tar.gz: 07fe0fcd40f444925cc19e080bbf942459ef85bb
+SHA512:
+  metadata.gz: 29d1063e514fb7c48335b86ab5eb78b2433b463e0869a64d6338dce96ddc1ebcb2e1350584ee0dbb65277adfdced6dfc1504df649c90a4748663480565f6b32b
+  data.tar.gz: 077b8961c275cf7277f93d70d2220bdfff7abbf5a2094dffa7e774199cbe618ca57f9f9707999d158c0fd48982655c2fde206848e5b3e6cff5ca67b9443332d6

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 0.10.42 - 2013/12/25
+
+* parser/in_tail: Add 'types' and related parameters to convert record value type
+* in_syslog: Add 'protocol_type' option for TCP support.
+* Buffer: Extract 'storable?' method for hook judge mechanizm by 3rd-party buffer plugins.
+* Output: Add 'try_flush_interval' parameter for high frequency flush.
+  It is undocumented because almost users never need this feature.
+* Change configuration unused checking to prevent memory leak when a plugin accesses configuratino keys.
+
 Release 0.10.41 - 2013/12/05
 
 * BufferedOutput: Add 'max_retry_wait' to limit retry interval

data/lib/fluent/buffer.rb

@@ -158,13 +158,17 @@ module Fluent
       end
     end
 
+    def storable?(chunk, data)
+      chunk.size + data.bytesize <= @buffer_chunk_limit
+    end
+
     def emit(key, data, chain)
       key = key.to_s
 
       synchronize do
         top = (@map[key] ||= new_chunk(key))  # TODO generate unique chunk id
 
-        if top.size + data.bytesize <= @buffer_chunk_limit
+        if storable?(top, data)
           chain.next
           top << data
           return false

data/lib/fluent/config.rb

@@ -25,7 +25,7 @@ module Fluent
 
   module Config
     class Element < Hash
-      def initialize(name, arg, attrs, elements, used=[])
+      def initialize(name, arg, attrs, elements, unused=nil)
         @name = name
         @arg = arg
         @elements = elements
@@ -33,10 +33,10 @@ module Fluent
         attrs.each {|k,v|
           self[k] = v
         }
-        @used = used
+        @unused = unused || attrs.keys
       end
 
-      attr_accessor :name, :arg, :elements, :used
+      attr_accessor :name, :arg, :elements, :unused
 
       def add_element(name, arg='')
         e = Element.new(name, arg, {}, [])
@@ -45,22 +45,22 @@ module Fluent
       end
 
       def +(o)
-        Element.new(@name.dup, @arg.dup, o.merge(self), @elements+o.elements, @used+o.used)
+        Element.new(@name.dup, @arg.dup, o.merge(self), @elements+o.elements, (@unused+o.unused).uniq)
       end
 
       def has_key?(key)
-        @used << key
+        @unused.delete(key)
         super
       end
 
       def [](key)
-        @used << key
+        @unused.delete(key)
         super
       end
 
       def check_not_fetched(&block)
         each_key {|key|
-          unless @used.include?(key)
+          if @unused.include?(key)
             block.call(key, self)
           end
         }

data/lib/fluent/output.rb

@@ -172,6 +172,7 @@ module Fluent
 
     config_param :buffer_type, :string, :default => 'memory'
     config_param :flush_interval, :time, :default => 60
+    config_param :try_flush_interval, :float, :default => 1
     config_param :retry_limit, :integer, :default => 17
     config_param :retry_wait, :time, :default => 1.0
     config_param :max_retry_wait, :time, :default => nil
@@ -277,7 +278,7 @@ module Fluent
         end
       end
       if empty
-        return time + 1  # TODO 1
+        return time + @try_flush_interval
       end
 
       begin
@@ -288,7 +289,7 @@ module Fluent
             if retrying = !@error_history.empty?  # re-check in synchronize
               if @next_retry_time >= time
                 # allow retrying for only one thread
-                return time + 1  # TODO 1
+                return time + @try_flush_interval
               end
               # assume next retry failes and
               # clear them if when it succeeds
@@ -316,7 +317,7 @@ module Fluent
         if has_next
           return Engine.now + @queued_chunk_flush_interval
         else
-          return time + 1  # TODO 1
+          return time + @try_flush_interval
         end
 
       rescue => e

data/lib/fluent/parser.rb

@@ -51,8 +51,78 @@ module Fluent
       end
     end
 
+    module TypeConverter
+      Converters = {
+        'string' => lambda { |v| v.to_s },
+        'integer' => lambda { |v| v.to_i },
+        'float' => lambda { |v| v.to_f },
+        'bool' => lambda { |v|
+          case v.downcase
+          when 'true', 'yes', '1'
+            true
+          else
+            false
+          end
+        },
+        'time' => lambda { |v, time_parser|
+          time_parser.parse(v)
+        },
+        'array' => lambda { |v, delimiter|
+          v.to_s.split(delimiter)
+        }
+      }
+
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :types, :string, :default => nil
+          config_param :types_delimiter, :string, :default => ','
+          config_param :types_label_delimiter, :string, :default => ':'
+        }
+      end
+
+      def configure(conf)
+        super
+
+        @type_converters = parse_types_parameter unless @types.nil?
+      end
+
+      private
+
+      def convert_type(name, value)
+        converter = @type_converters[name]
+        converter.nil? ? value : converter.call(value)
+      end
+
+      def parse_types_parameter
+        converters = {}
+
+        @types.split(@types_delimiter).each { |pattern_name|
+          name, type, format = pattern_name.split(@types_label_delimiter, 3)
+          raise ConfigError, "Type is needed" if type.nil?
+
+          case type
+          when 'time'
+            t_parser = TimeParser.new(format)
+            converters[name] = lambda { |v|
+              Converters[type].call(v, t_parser)
+            }
+          when 'array'
+            delimiter = format || ','
+            converters[name] = lambda { |v|
+              Converters[type].call(v, delimiter)
+            }
+          else
+            converters[name] = Converters[type]
+          end
+        }
+
+        converters
+      end
+    end
+
     class RegexpParser
       include Configurable
+      include TypeConverter
 
       config_param :time_format, :string, :default => nil
 
@@ -83,7 +153,11 @@ module Fluent
             when "time"
               time = @mutex.synchronize { @time_parser.parse(value) }
             else
-              record[name] = value
+              record[name] = if @type_converters.nil?
+                               value
+                             else
+                               convert_type(name, value)
+                             end
             end
           end
         }
@@ -131,6 +205,7 @@ module Fluent
 
     class ValuesParser
       include Configurable
+      include TypeConverter
 
       config_param :keys, :string
       config_param :time_key, :string, :default => nil
@@ -163,8 +238,18 @@ module Fluent
           time = Engine.now
         end
 
+        convert_field_type!(record) if @type_converters
+
         return time, record
       end
+
+      private
+
+      def convert_field_type!(record)
+        record.each { |key, value|
+          record[key] = convert_type(key, value)
+        }
+      end
     end
 
     class TSVParser < ValuesParser

data/lib/fluent/plugin/in_syslog.rb

@@ -63,12 +63,23 @@ module Fluent
 
     def initialize
       super
+      require 'cool.io'
       require 'fluent/plugin/socket_util'
     end
 
     config_param :port, :integer, :default => 5140
     config_param :bind, :string, :default => '0.0.0.0'
     config_param :tag, :string
+    config_param :protocol_type, :default => :udp do |val|
+      case val.downcase
+      when 'tcp'
+        :tcp
+      when 'udp'
+        :udp
+      else
+        raise ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
+      end
+    end
 
     def configure(conf)
       super
@@ -78,6 +89,7 @@ module Fluent
         @parser = parser
       else
         @parser = nil
+        @time_parser = TextParser::TimeParser.new(TIME_FORMAT)
       end
     end
 
@@ -89,12 +101,7 @@ module Fluent
       end
 
       @loop = Coolio::Loop.new
-
-      $log.debug "listening syslog socket on #{@bind}:#{@port}"
-      @usock = SocketUtil.create_udp_socket(@bind)
-      @usock.bind(@bind, @port)
-
-      @handler = UdpHandler.new(@usock, callback)
+      @handler = listen(callback)
       @loop.attach(@handler)
 
       @thread = Thread.new(&method(:run))
@@ -153,7 +160,7 @@ module Fluent
           when "pri"
             pri = value.to_i
           when "time"
-            time = Time.strptime(value.gsub(/ +/, ' '), TIME_FORMAT).to_i
+            time = @time_parser.parse(value.gsub(/ +/, ' '))
           else
             record[name] = value
           end
@@ -170,6 +177,18 @@ module Fluent
     end
 
     private
+
+    def listen(callback)
+      $log.debug "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
+      if @protocol_type == :udp
+        @usock = SocketUtil.create_udp_socket(@bind)
+        @usock.bind(@bind, @port)
+        UdpHandler.new(@usock, callback)
+      else
+        Coolio::TCPServer.new(@bind, @port, TcpHandler, callback)
+      end
+    end
+
     def emit(pri, time, record)
       facility = FACILITY_MAP[pri >> 3]
       priority = PRIORITY_MAP[pri & 0b111]
@@ -198,5 +217,41 @@ module Fluent
         # TODO log?
       end
     end
+
+    class TcpHandler < Coolio::Socket
+      def initialize(io, on_message)
+        super(io)
+        if io.is_a?(TCPSocket)
+          opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+        end
+        $log.trace { "accepted fluent socket object_id=#{self.object_id}" }
+        @on_message = on_message
+        @buffer = "".force_encoding('ASCII-8BIT')
+      end
+
+      def on_connect
+      end
+
+      def on_read(data)
+        @buffer << data
+        pos = 0
+
+        # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+        while i = @buffer.index("\n", pos)
+          msg = @buffer[pos..i]
+          @on_message.call(msg)
+          pos = i + 1
+        end
+        @buffer.slice!(0, pos) if pos > 0
+      rescue => e
+        $log.error "syslog error", :error => e, :error_class => e.class
+        close
+      end
+
+      def on_close
+        $log.trace { "closed fluent socket object_id=#{self.object_id}" }
+      end
+    end
   end
 end

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.41'
+  VERSION = '0.10.42'
 
 end

data/test/config.rb

@@ -58,6 +58,50 @@ class ConfigTest < Test::Unit::TestCase
     
   end
 
+  def test_check_not_fetchd
+    write_config "#{TMP_DIR}/config_test_not_fetched.conf", %[
+      <match dummy>
+       type          rewrite
+       add_prefix    filtered
+       <rule>
+         key     path
+         pattern ^[A-Z]+
+         replace
+       </rule>
+     </match>
+    ]
+    root_conf  = Config.read("#{TMP_DIR}/config_test_not_fetched.conf")
+    match_conf = root_conf.elements.first
+    rule_conf  = match_conf.elements.first
+
+    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[type add_prefix key pattern replace], not_fetched
+
+    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[type add_prefix key pattern replace], not_fetched
+
+    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[key pattern replace], not_fetched
+
+    # accessing should delete
+    match_conf['type']
+    rule_conf['key']
+
+    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[add_prefix pattern replace], not_fetched
+
+    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[add_prefix pattern replace], not_fetched
+
+    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[pattern replace], not_fetched
+
+    # repeateadly accessing should not grow memory usage
+    before_size = match_conf.unused.size
+    10.times { match_conf['type'] }
+    assert_equal before_size, match_conf.unused.size
+  end
+
   def write_config(path, data)
     FileUtils.mkdir_p(File.dirname(path))
     File.open(path, "w") {|f| f.write data }

data/test/parser.rb

@@ -41,6 +41,34 @@ module ParserTest
     end
   end
 
+  class RegexpParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def internal_test_case(parser)
+      time, record = parser.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] [14/Feb/2013:12:00:00 +0900] "true /,/user HTTP/1.1" 200 777')
+
+      assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal({
+        'user' => '-',
+        'flag' => true,
+        'code' => 200.0,
+        'size' => 777,
+        'date' => str2time('14/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'),
+        'host' => '192.168.0.1',
+        'path' => ['/', '/user']
+      }, record)
+    end
+
+    def test_call_with_typed
+      # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0 
+      internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'))
+    end
+
+    def test_call_with_typed_and_name_separator
+      internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
+    end
+  end
+
   class ApacheParserTest < ::Test::Unit::TestCase
     include ParserTest
 

data/test/plugin/in_syslog.rb

@@ -65,11 +65,7 @@ class SyslogInputTest < Test::Unit::TestCase
 
   def test_msg_size
     d = create_driver
-
-    tests = [
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100, 'expected' => 'x' * 100},
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024, 'expected' => 'x' * 1024},
-    ]
+    tests = create_test_case
 
     d.run do
       u = UDPSocket.new
@@ -80,10 +76,52 @@ class SyslogInputTest < Test::Unit::TestCase
       sleep 1
     end
 
-    emits = d.emits
+    compare_test_result(d.emits, tests)
+  end
+
+  def test_msg_size_with_tcp
+    d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
+    tests = create_test_case
+
+    d.run do
+      tests.each {|test|
+        TCPSocket.open('127.0.0.1', PORT) do |s|
+          s.send(test['msg'], 0)
+        end
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
+  def test_msg_size_with_same_tcp_connection
+    d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
+    tests = create_test_case
+
+    d.run do
+      TCPSocket.open('127.0.0.1', PORT) do |s|
+        tests.each {|test|
+          s.send(test['msg'], 0)
+        }
+      end
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
+  def create_test_case
+    # actual syslog message has "\n"
+    [
+      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
+    ]
+  end
+
+  def compare_test_result(emits, tests)
     emits.each_index {|i|
       assert_equal(tests[i]['expected'], emits[i][2]['message'])
     }
   end
 end
-

metadata

@@ -1,34 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.41
-  prerelease: 
+  version: 0.10.42
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-04 00:00:00.000000000 Z
+date: 2013-12-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -37,21 +35,20 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - ! '!='
+    - - '!='
       - !ruby/object:Gem::Version
         version: 0.5.3
     - - <
@@ -60,23 +57,20 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.3
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -84,7 +78,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -92,7 +85,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: cool.io
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -100,7 +92,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -108,7 +99,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -116,7 +106,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -124,65 +113,57 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: parallel_tests
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.15.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.15.3
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.3.0
 description: Fluentd is an event collector system. It is a generalized version of
@@ -290,30 +271,26 @@ files:
 - test/scripts/exec_script.rb
 homepage: http://fluentd.org/
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -602194787130919488
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fluentd event collector
 test_files:
 - test/config.rb