fluentd 0.14.1 → 0.14.2

data/lib/fluent/plugin/parser.rb

@@ -97,75 +97,6 @@ module Fluent
       end
     end
 
-    class RegexpParser < Parser
-      include Fluent::TypeConverter
-
-      config_param :time_key, :string, default: 'time'
-      config_param :time_format, :string, default: nil
-
-      def initialize(regexp, conf={})
-        super()
-
-        unless conf.empty?
-          unless conf.is_a?(Config::Element)
-            conf = Config::Element.new('default_regexp_conf', '', conf, [])
-          end
-          configure(conf)
-        end
-
-        @regexp = regexp
-        @time_parser = TimeParser.new(@time_format)
-        @mutex = Mutex.new
-      end
-
-      def configure(conf)
-        super
-        @time_parser = TimeParser.new(@time_format)
-      end
-
-      def patterns
-        {'format' => @regexp, 'time_format' => @time_format}
-      end
-
-      def parse(text)
-        m = @regexp.match(text)
-        unless m
-          yield nil, nil
-          return
-        end
-
-        time = nil
-        record = {}
-
-        m.names.each do |name|
-          if value = m[name]
-            if name == @time_key
-              time = @mutex.synchronize { @time_parser.parse(value) }
-              if @keep_time_key
-                record[name] = if @type_converters.nil?
-                                 value
-                               else
-                                 convert_type(name, value)
-                               end
-              end
-            else
-              record[name] = if @type_converters.nil?
-                               value
-                             else
-                               convert_type(name, value)
-                             end
-            end
-          end
-        end
-
-        if @estimate_current_event
-          time ||= Fluent::EventTime.now
-        end
-
-        yield time, record
-      end
-    end
-
     class ValuesParser < Parser
       include Fluent::TypeConverter
 

data/lib/fluent/plugin/parser_apache.rb

@@ -16,9 +16,13 @@
 
 require 'fluent/plugin/parser'
 
-Fluent::Plugin.register_parser('apache', Proc.new{
-  Fluent::Plugin::RegexpParser.new(
-    /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,
-    {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}
-  )
-})
+module Fluent
+  module Plugin
+    class ApacheParser < RegexpParser
+      Plugin.register_parser("apache", self)
+
+      config_set_default :expression, %q{/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
+    end
+  end
+end

data/lib/fluent/plugin/parser_apache_error.rb

@@ -16,6 +16,11 @@
 
 require 'fluent/plugin/parser'
 
-Fluent::Plugin.register_parser('apache_error', Proc.new {
-  Fluent::Plugin::RegexpParser.new(/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/)
-})
+module Fluent
+  module Plugin
+    class ApacheErrorParser < RegexpParser
+      Plugin.register_parser("apache_error", self)
+      config_set_default :expression, %q{/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/}
+    end
+  end
+end

data/lib/fluent/plugin/parser_csv.rb

@@ -23,8 +23,10 @@ module Fluent
     class CSVParser < ValuesParser
       Plugin.register_parser('csv', self)
 
+      config_param :delimiter, :string, default: ','
+
       def parse(text)
-        yield values_map(CSV.parse_line(text))
+        yield values_map(CSV.parse_line(text, col_sep: @delimiter))
       end
     end
   end

data/lib/fluent/plugin/parser_json.rb

@@ -39,7 +39,7 @@ module Fluent
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {bigdecimal_load: :float}
+          Oj.default_options = {bigdecimal_load: :float, mode: :strict}
           @load_proc = Oj.method(:load)
           @error_class = Oj::ParseError
         rescue LoadError

data/lib/fluent/plugin/parser_multiline.rb

@@ -30,11 +30,13 @@ module Fluent
 
         formats = parse_formats(conf).compact.map { |f| f[1..-2] }.join
         begin
-          @regex = Regexp.new(formats, Regexp::MULTILINE)
-          if @regex.named_captures.empty?
+          regexp = Regexp.new(formats, Regexp::MULTILINE)
+          if regexp.named_captures.empty?
             raise "No named captures"
           end
-          @parser = RegexpParser.new(@regex, conf)
+          regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/", "multiline" => true }, [])
+          @parser = Fluent::Plugin::RegexpParser.new
+          @parser.configure(conf + regexp_conf)
         rescue => e
           raise ConfigError, "Invalid regexp '#{formats}': #{e}"
         end

data/lib/fluent/plugin/parser_nginx.rb

@@ -16,9 +16,13 @@
 
 require 'fluent/plugin/parser'
 
-Fluent::Plugin.register_parser('nginx', Proc.new {
-  Fluent::Plugin::RegexpParser.new(
-    /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,
-    {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}
-  )
-})
+module Fluent
+  module Plugin
+    class NginxParser < RegexpParser
+      Plugin.register_parser("nginx", self)
+
+      config_set_default :expression, %q{/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
+    end
+  end
+end

data/lib/fluent/plugin/parser_regexp.rb

@@ -0,0 +1,73 @@
+module Fluent
+  module Plugin
+    class RegexpParser < Parser
+      include Fluent::Compat::TypeConverter
+
+      Plugin.register_parser("regexp", self)
+
+      config_param :expression, :string, default: ""
+      config_param :ignorecase, :bool, default: false
+      config_param :multiline, :bool, default: false
+      config_param :time_key, :string, default: 'time'
+      config_param :time_format, :string, default: nil
+
+      def initialize
+        super
+        @mutex = Mutex.new
+      end
+
+      def configure(conf)
+        super
+        @time_parser = TimeParser.new(@time_format)
+        unless @expression.empty?
+          if @expression[0] == "/" && @expression[-1] == "/"
+            regexp_option = 0
+            regexp_option |= Regexp::IGNORECASE if @ignorecase
+            regexp_option |= Regexp::MULTILINE if @multiline
+            @regexp = Regexp.new(@expression[1..-2], regexp_option)
+          else
+            raise Fluent::ConfigError, "expression must start with `/` and end with `/`: #{@expression}"
+          end
+        end
+      end
+
+      def parse(text)
+        m = @regexp.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        time = nil
+        record = {}
+
+        m.names.each do |name|
+          if value = m[name]
+            if name == @time_key
+              time = @mutex.synchronize { @time_parser.parse(value) }
+              if @keep_time_key
+                record[name] = if @type_converters.nil?
+                                 value
+                               else
+                                 convert_type(name, value)
+                               end
+              end
+            else
+              record[name] = if @type_converters.nil?
+                               value
+                             else
+                               convert_type(name, value)
+                             end
+            end
+          end
+        end
+
+        if @estimate_current_event
+          time ||= Fluent::EventTime.now
+        end
+
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/socket_util.rb

@@ -14,155 +14,9 @@
 #    limitations under the License.
 #
 
-require 'ipaddr'
-
-require 'cool.io'
-
-require 'fluent/plugin'
-require 'fluent/input'
+require 'fluent/compat/socket_util'
 
 module Fluent
-  module Plugin
-    module SocketUtil
-      def create_udp_socket(host)
-        if IPAddr.new(IPSocket.getaddress(host)).ipv4?
-          UDPSocket.new
-        else
-          UDPSocket.new(Socket::AF_INET6)
-        end
-      end
-      module_function :create_udp_socket
-
-      class UdpHandler < Coolio::IO
-        def initialize(io, log, body_size_limit, callback)
-          super(io)
-          @io = io
-          @log = log
-          @body_size_limit = body_size_limit
-          @callback = callback
-        end
-
-        def on_readable
-          msg, addr = @io.recvfrom_nonblock(@body_size_limit)
-          msg.chomp!
-          @callback.call(msg, addr)
-        rescue => e
-          @log.error "unexpected error", error: e
-        end
-      end
-
-      class TcpHandler < Coolio::Socket
-        PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
-
-        def initialize(io, log, delimiter, callback)
-          super(io)
-          @timeout = 0
-          if io.is_a?(TCPSocket)
-            @addr = (io.peeraddr rescue PEERADDR_FAILED)
-
-            opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-            io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-          end
-          @delimiter = delimiter
-          @callback = callback
-          @log = log
-          @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
-          @buffer = "".force_encoding('ASCII-8BIT')
-        end
-
-        def on_connect
-        end
-
-        def on_read(data)
-          @buffer << data
-          pos = 0
-
-          while i = @buffer.index(@delimiter, pos)
-            msg = @buffer[pos...i]
-            @callback.call(msg, @addr)
-            pos = i + @delimiter.length
-          end
-          @buffer.slice!(0, pos) if pos > 0
-        rescue => e
-          @log.error "unexpected error", error: e
-          close
-        end
-
-        def on_close
-          @log.trace { "closed fluent socket object_id=#{self.object_id}" }
-        end
-      end
-
-      class BaseInput < Fluent::Input
-        def initialize
-          super
-          require 'fluent/parser'
-        end
-
-        desc 'Tag of output events.'
-        config_param :tag, :string
-        desc 'The format of the payload.'
-        config_param :format, :string
-        desc 'The port to listen to.'
-        config_param :port, :integer, default: 5150
-        desc 'The bind address to listen to.'
-        config_param :bind, :string, default: '0.0.0.0'
-        desc "The field name of the client's hostname."
-        config_param :source_host_key, :string, default: nil
-        config_param :blocking_timeout, :time, default: 0.5
-
-        def configure(conf)
-          super
-
-          @parser = Plugin.new_parser(@format)
-          @parser.configure(conf)
-        end
-
-        def start
-          super
-
-          @loop = Coolio::Loop.new
-          @handler = listen(method(:on_message))
-          @loop.attach(@handler)
-          @thread = Thread.new(&method(:run))
-        end
-
-        def shutdown
-          @loop.watchers.each { |w| w.detach }
-          @loop.stop if @loop.instance_variable_get("@running")
-          @handler.close
-          @thread.join
-
-          super
-        end
-
-        def run
-          @loop.run(@blocking_timeout)
-        rescue => e
-          log.error "unexpected error", error: e
-          log.error_backtrace
-        end
-
-        private
-
-        def on_message(msg, addr)
-          @parser.parse(msg) { |time, record|
-            unless time && record
-              log.warn "pattern not match: #{msg.inspect}"
-              return
-            end
-
-            record[@source_host_key] = addr[3] if @source_host_key
-            router.emit(@tag, time, record)
-          }
-        rescue => e
-          log.error msg.dump, error: e, host: addr[3]
-          log.error_backtrace
-        end
-      end
-    end
-  end
-
   # obsolete
-  SocketUtil = Fluent::Plugin::SocketUtil
+  SocketUtil = Fluent::Compat::SocketUtil
 end

data/lib/fluent/plugin/storage_local.rb

@@ -58,7 +58,7 @@ module Fluent
               raise Fluent::ConfigError, "Unexpected error: failed to read data from plugin storage file: '#{@path}'"
             end
           else
-            raise Fluent::ConfigError, "Directory is not writable for plugin storage file '#{@path}'" unless File.writable?(@path)
+            raise Fluent::ConfigError, "Directory is not writable for plugin storage file '#{dir}'" unless File.writable?(dir)
           end
         end
       end

data/lib/fluent/plugin/string_util.rb

@@ -14,24 +14,9 @@
 #    limitations under the License.
 #
 
-module Fluent
-  module Plugin
-    module StringUtil
-      def match_regexp(regexp, string)
-        begin
-          return regexp.match(string)
-        rescue ArgumentError => e
-          raise e unless e.message.index("invalid byte sequence in".freeze).zero?
-          $log.info "invalid byte sequence is replaced in `#{string}`"
-          string = string.scrub('?')
-          retry
-        end
-        return true
-      end
-      module_function :match_regexp
-    end
-  end
+require 'fluent/compat/string_util'
 
+module Fluent
   # obsolete
-  StringUtil = Fluent::Plugin::StringUtil
+  StringUtil = Fluent::Compat::StringUtil
 end

data/lib/fluent/plugin_helper.rb

@@ -22,6 +22,7 @@ require 'fluent/plugin_helper/child_process'
 require 'fluent/plugin_helper/storage'
 require 'fluent/plugin_helper/parser'
 require 'fluent/plugin_helper/formatter'
+require 'fluent/plugin_helper/inject'
 require 'fluent/plugin_helper/retry_state'
 require 'fluent/plugin_helper/compat_parameters'