fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/in_syslog.rb

@@ -14,17 +14,19 @@
 #    limitations under the License.
 #
 
-require 'cool.io'
-require 'yajl'
-
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/config/error'
-require 'fluent/parser'
+require 'fluent/plugin/parser'
+
+require 'yajl'
 
-module Fluent
+module Fluent::Plugin
   class SyslogInput < Input
-    Plugin.register_input('syslog', self)
+    Fluent::Plugin.register_input('syslog', self)
 
+    helpers :parser, :compat_parameters, :server
+
+    DEFAULT_PARSER = 'syslog'
     SYSLOG_REGEXP = /^\<([0-9]+)\>(.*)/
 
     FACILITY_MAP = {
@@ -65,11 +67,6 @@ module Fluent
       7  => 'debug'
     }
 
-    def initialize
-      super
-      require 'fluent/plugin/socket_util'
-    end
-
     desc 'The port to listen to.'
     config_param :port, :integer, default: 5140
     desc 'The bind address to listen to.'
@@ -77,188 +74,169 @@ module Fluent
     desc 'The prefix of the tag. The tag itself is generated by the tag prefix, facility level, and priority.'
     config_param :tag, :string
     desc 'The transport protocol used to receive logs.(udp, tcp)'
-    config_param :protocol_type, default: :udp do |val|
-      case val.downcase
-      when 'tcp'
-        :tcp
-      when 'udp'
-        :udp
-      else
-        raise ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
-      end
-    end
+    config_param :protocol_type, :enum, list: [:tcp, :udp], default: nil, deprecated: "use transport directive"
+    desc 'The message frame type.(traditional, octet_count)'
+    config_param :frame_type, :enum, list: [:traditional, :octet_count], default: :traditional
+
     desc 'If true, add source host to event record.'
-    config_param :include_source_host, :bool, default: false, deprecated: "use source_hostname_key instead"
+    config_param :include_source_host, :bool, default: false, deprecated: 'use "source_hostname_key" or "source_address_key" instead.'
     desc 'Specify key of source host when include_source_host is true.'
-    config_param :source_host_key, :string, default: 'source_host'.freeze, deprecated: "use source_hostname_key instead"
-    desc "The field name of the client's hostname."
+    config_param :source_host_key, :string, default: 'source_host'.freeze
+
+    desc 'The field name of hostname of sender.'
     config_param :source_hostname_key, :string, default: nil
-    desc "The field name of the client's source address."
-    config_param :source_address_key, :string, default: nil
     desc 'Try to resolve hostname from IP addresses or not.'
     config_param :resolve_hostname, :bool, default: nil
+    desc 'The field name of source address of sender.'
+    config_param :source_address_key, :string, default: nil
     desc 'The field name of the priority.'
     config_param :priority_key, :string, default: nil
     desc 'The field name of the facility.'
     config_param :facility_key, :string, default: nil
+
     desc "The max bytes of message"
     config_param :message_length_limit, :size, default: 2048
+
     config_param :blocking_timeout, :time, default: 0.5
 
-    desc 'If true, accept syslog message without PRI part'
-    config_param :allow_without_priority, :bool, default: false
-    # 13 is the default value of rsyslog and syslog-ng
-    desc 'The default PRI value (0 - 191 are available)'
-    config_param :default_priority, :integer, default: 13
+    desc 'The delimiter value "\n"'
+    config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message
+
+    config_section :parse do
+      config_set_default :@type, DEFAULT_PARSER
+      config_param :with_priority, :bool, default: true
+    end
+
+    # overwrite server plugin to change default to :udp
+    config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
+      config_argument :protocol, :enum, list: [:tcp, :udp, :tls], default: :udp
+    end
 
     def configure(conf)
+      compat_parameters_convert(conf, :parser)
+
       super
 
-      if @default_priority < 0 && @default_priority > 191
-        raise ConfigError, "default_priority must be 0 ~ 191"
-      end
+      @use_default = false
 
-      if @allow_without_priority && conf['message_format'] == 'auto'
-        raise ConfigError, "message_format auto isn't allowed when allow_without_priority is true"
-      end
+      @parser = parser_create
+      @parser_parse_priority = @parser.respond_to?(:with_priority) && @parser.with_priority
 
-      conf['with_priority'] = !@allow_without_priority
-      if conf.has_key?('format')
-        @parser = Plugin.new_parser(conf['format'])
-        @parser.configure(conf)
-      else
-        @parser = TextParser::SyslogParser.new
-        @parser.configure(conf)
-        @use_default = true
-      end
-
-      if @source_hostname_key.nil? && @include_source_host
-        @source_hostname_key = @source_host_key
+      if @include_source_host
+        if @source_address_key
+          raise Fluent::ConfigError, "specify either source_address_key or include_source_host"
+        end
+        @source_address_key = @source_host_key
       end
       if @source_hostname_key
         if @resolve_hostname.nil?
           @resolve_hostname = true
-        elsif !@resolve_hostname # user specifies "false" in configure
+        elsif !@resolve_hostname # user specifies "false" in config
           raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
         end
       end
-    end
-
-    def start
-      callback = if @allow_without_priority
-                   method(:receive_data_allow_without_priority)
-                 else
-                   if @use_default
-                     method(:receive_data_default)
-                   else
-                     method(:receive_data_with_format)
-                   end
-                 end
-      @loop = Coolio::Loop.new
-      @handler = listen(callback)
-      @loop.attach(@handler)
-
-      @thread = Thread.new(&method(:run))
-    end
 
-    def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @handler.close
-      @thread.join
+      @_event_loop_run_timeout = @blocking_timeout
     end
 
-    def run
-      @loop.run(@blocking_timeout)
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
+    def multi_workers_ready?
+      true
     end
 
-    private
+    def start
+      super
 
-    ## SyslogParser with PRI part
-    def receive_data_default(data, addr)
-      parse_text(data, addr)
-    rescue => e
-      log.error data.dump, error: e.to_s
-      log.error_backtrace
+      log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type || @transport_config.protocol}"
+      case @protocol_type || @transport_config.protocol
+      when :udp then start_udp_server
+      when :tcp then start_tcp_server
+      when :tls then start_tcp_server(tls: true)
+      else
+        raise "BUG: invalid transport value: #{@protocol_type || @transport_config.protocol}"
+      end
     end
 
-    ## PRI part parser + custom parser without PRI part
-    def receive_data_with_format(data, addr)
-      m = SYSLOG_REGEXP.match(data)
-      unless m
-        log.warn "invalid syslog message: #{data.dump}"
-        return
+    def start_udp_server
+      server_create_udp(:in_syslog_udp_server, @port, bind: @bind, max_bytes: @message_length_limit, resolve_name: @resolve_hostname) do |data, sock|
+        message_handler(data.chomp, sock)
       end
-      pri = m[1].to_i
-      text = m[2]
+    end
 
-      parse_text(text, addr, pri)
-    rescue => e
-      log.error data.dump, error: e.to_s
-      log.error_backtrace
+    def start_tcp_server(tls: false)
+      octet_count_frame = @frame_type == :octet_count
+
+      delimiter = octet_count_frame ? " " : @delimiter
+      delimiter_size = delimiter.size
+      server_create_connection(tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
+        conn.data do |data|
+          buffer = conn.buffer
+          buffer << data
+          pos = 0
+          if octet_count_frame
+            while idx = buffer.index(delimiter, pos)
+              num = Integer(buffer[pos..idx])
+              pos = idx + num
+              msg = buffer[idx + 1...pos]
+              if msg.size < num - 1
+                pos = pos - num - num.to_s.size
+                break
+              end
+              message_handler(msg, conn)
+            end
+          else
+            while idx = buffer.index(delimiter, pos)
+              msg = buffer[pos...idx]
+              pos = idx + delimiter_size
+              message_handler(msg, conn)
+            end
+          end
+          buffer.slice!(0, pos) if pos > 0
+        end
+      end
     end
 
-    ## PRI part parser + SyslogParser without PRI part | custom parser without PRI part
-    def receive_data_allow_without_priority(data, addr)
-      m = SYSLOG_REGEXP.match(data)
-      if m
+    private
+
+    def message_handler(data, sock)
+      pri = nil
+      text = data
+      unless @parser_parse_priority
+        m = SYSLOG_REGEXP.match(data)
+        unless m
+          log.warn "invalid syslog message: #{data.dump}"
+          return
+        end
         pri = m[1].to_i
         text = m[2]
-      else
-        pri = @default_priority
-        text = data
       end
 
-      parse_text(text, addr, pri)
-    rescue => e
-      log.error data.dump, error: e.to_s
-      log.error_backtrace
-    end
-
-    def parse_text(text, addr, pri = nil)
-      @parser.parse(text) { |time, record|
+      @parser.parse(text) do |time, record|
         unless time && record
-          log.warn "pattern not match: #{text.inspect}"
+          log.warn "failed to parse message", data: data
           return
         end
 
-        ## from receive_data_default
-        pri = record.delete('pri'.freeze) unless pri
-
+        pri ||= record.delete('pri')
         facility = FACILITY_MAP[pri >> 3]
         priority = PRIORITY_MAP[pri & 0b111]
 
         record[@priority_key] = priority if @priority_key
         record[@facility_key] = facility if @facility_key
-        record[@source_hostname_key] = addr[2] if @source_hostname_key
-        record[@source_address_key] = addr[3] if @source_address_key
+        record[@source_address_key] = sock.remote_addr if @source_address_key
+        record[@source_hostname_key] = sock.remote_host if @source_hostname_key
 
         tag = "#{@tag}.#{facility}.#{priority}"
         emit(tag, time, record)
-      }
-    end
-
-    private
-
-    def listen(callback)
-      log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
-      if @protocol_type == :udp
-        @usock = SocketUtil.create_udp_socket(@bind)
-        @usock.bind(@bind, @port)
-        SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, @resolve_hostname)
-      else
-        # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
-        Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, "\n", callback, @resolve_hostname)
       end
+    rescue => e
+      log.error "invalid input", data: data, error: e
+      log.error_backtrace
     end
 
     def emit(tag, time, record)
       router.emit(tag, time, record)
     rescue => e
-      log.error "syslog failed to emit", error: e.to_s, error_class: e.class.to_s, tag: tag, record: Yajl.dump(record)
+      log.error "syslog failed to emit", error: e, tag: tag, record: Yajl.dump(record)
     end
   end
 end