fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/parser_multiline.rb

@@ -0,0 +1,106 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class MultilineParser < Parser
+      Plugin.register_parser('multiline', self)
+
+      desc 'Specify regexp pattern for start line of multiple lines'
+      config_param :format_firstline, :string, default: nil
+
+      FORMAT_MAX_NUM = 20
+
+      def configure(conf)
+        super
+
+        formats = parse_formats(conf).compact.map { |f| f[1..-2] }.join
+        begin
+          regexp = Regexp.new(formats, Regexp::MULTILINE)
+          if regexp.named_captures.empty?
+            raise "No named captures"
+          end
+          regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/m" }, [])
+          @parser = Fluent::Plugin::RegexpParser.new
+          @parser.configure(conf + regexp_conf)
+        rescue => e
+          raise Fluent::ConfigError, "Invalid regexp '#{formats}': #{e}"
+        end
+
+        if @format_firstline
+          check_format_regexp(@format_firstline, 'format_firstline')
+          @firstline_regex = Regexp.new(@format_firstline[1..-2])
+        end
+      end
+
+      def parse(text, &block)
+        @parser.call(text, &block)
+      end
+
+      def has_firstline?
+        !!@format_firstline
+      end
+
+      def firstline?(text)
+        @firstline_regex.match(text)
+      end
+
+      private
+
+      def parse_formats(conf)
+        check_format_range(conf)
+
+        prev_format = nil
+        (1..FORMAT_MAX_NUM).map { |i|
+          format = conf["format#{i}"]
+          if (i > 1) && prev_format.nil? && !format.nil?
+            raise Fluent::ConfigError, "Jump of format index found. format#{i - 1} is missing."
+          end
+          prev_format = format
+          next if format.nil?
+
+          check_format_regexp(format, "format#{i}")
+          format
+        }
+      end
+
+      def check_format_range(conf)
+        invalid_formats = conf.keys.select { |k|
+          m = k.match(/^format(\d+)$/)
+          m ? !((1..FORMAT_MAX_NUM).include?(m[1].to_i)) : false
+        }
+        unless invalid_formats.empty?
+          raise Fluent::ConfigError, "Invalid formatN found. N should be 1 - #{FORMAT_MAX_NUM}: " + invalid_formats.join(",")
+        end
+      end
+
+      def check_format_regexp(format, key)
+        if format[0] == '/' && format[-1] == '/'
+          begin
+            Regexp.new(format[1..-2], Regexp::MULTILINE)
+          rescue => e
+            raise Fluent::ConfigError, "Invalid regexp in #{key}: #{e}"
+          end
+        else
+          raise Fluent::ConfigError, "format should be Regexp, need //, in #{key}: '#{format}'"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_nginx.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class NginxParser < RegexpParser
+      Plugin.register_parser("nginx", self)
+
+      config_set_default :expression, /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+\"?(?<http_x_forwarded_for>[^\"]*)\"?)?)?$/
+      config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
+    end
+  end
+end

data/lib/fluent/plugin/parser_none.rb

@@ -0,0 +1,36 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+require 'fluent/time'
+
+module Fluent
+  module Plugin
+    class NoneParser < Parser
+      Plugin.register_parser('none', self)
+
+      desc 'Field name to contain logs'
+      config_param :message_key, :string, default: 'message'
+
+      def parse(text)
+        record = {@message_key => text}
+        time = @estimate_current_event ? Fluent::EventTime.now : nil
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_regexp.rb

@@ -0,0 +1,68 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class RegexpParser < Parser
+      Plugin.register_parser("regexp", self)
+
+      desc 'Regular expression for matching logs'
+      config_param :expression, :regexp
+      desc 'Ignore case in matching'
+      config_param :ignorecase, :bool, default: false, deprecated: "Use /pattern/i instead, this option is no longer effective"
+      desc 'Build regular expression as a multline mode'
+      config_param :multiline, :bool, default: false, deprecated: "Use /pattern/m instead, this option is no longer effective"
+
+      config_set_default :time_key, 'time'
+
+      def configure(conf)
+        super
+        # For compat layer
+        if @ignorecase || @multiline
+          options = 0
+          options |= Regexp::IGNORECASE if @ignorecase
+          options |= Regexp::MULTILINE if @multiline
+          @expression = Regexp.compile(@expression.source, options)
+        end
+        @regexp = @expression # For backward compatibility
+
+        if @expression.named_captures.empty?
+          raise Fluent::ConfigError, "No named captures in 'expression' parameter. The regexp must have at least one named capture"
+        end
+      end
+
+      def parse(text)
+        m = @expression.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        r = {}
+        m.names.each do |name|
+          if value = m[name]
+            r[name] = value
+          end
+        end
+
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_syslog.rb

@@ -0,0 +1,142 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+require 'fluent/time'
+
+module Fluent
+  module Plugin
+    class SyslogParser < Parser
+      Plugin.register_parser('syslog', self)
+
+      # From existence TextParser pattern
+      REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      # From in_syslog default pattern
+      REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
+      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
+      REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
+
+      config_set_default :time_format, "%b %d %H:%M:%S"
+      desc 'If the incoming logs have priority prefix, e.g. <9>, set true'
+      config_param :with_priority, :bool, default: false
+      desc 'Specify protocol format'
+      config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
+      desc 'Specify time format for event time for rfc5424 protocol'
+      config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
+
+      def initialize
+        super
+        @mutex = Mutex.new
+      end
+
+      def configure(conf)
+        super
+
+        @time_parser_rfc3164 = @time_parser_rfc5424 = nil
+        @time_parser_rfc5424_without_subseconds = nil
+        @support_rfc5424_without_subseconds = false
+        @regexp = case @message_format
+                  when :rfc3164
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
+                    @with_priority ? REGEXP_WITH_PRI : REGEXP
+                  when :rfc5424
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
+                    @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
+                    @support_rfc5424_without_subseconds = true
+                    @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
+                  when :auto
+                    class << self
+                      alias_method :parse, :parse_auto
+                    end
+                    @time_parser_rfc3164 = time_parser_create(format: @time_format)
+                    @time_parser_rfc5424 = time_parser_create(format: @rfc5424_time_format)
+                    nil
+                  end
+        @time_parser = time_parser_create
+        @time_parser_rfc5424_without_subseconds = time_parser_create(format: "%Y-%m-%dT%H:%M:%S%z")
+      end
+
+      def patterns
+        {'format' => @regexp, 'time_format' => @time_format}
+      end
+
+      def parse(text)
+        # This is overwritten in configure
+      end
+
+      def parse_auto(text, &block)
+        if REGEXP_DETECT_RFC5424.match(text)
+          @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
+          @time_parser = @time_parser_rfc5424
+          @support_rfc5424_without_subseconds = true
+        else
+          @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+          @time_parser = @time_parser_rfc3164
+        end
+        parse_plain(text, &block)
+      end
+
+      def parse_plain(text, &block)
+        m = @regexp.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        time = nil
+        record = {}
+
+        m.names.each { |name|
+          if value = m[name]
+            case name
+            when "pri"
+              record['pri'] = value.to_i
+            when "time"
+              time = @mutex.synchronize do
+                time_str = value.squeeze(' ')
+                begin
+                  @time_parser.parse(time_str)
+                rescue Fluent::TimeParser::TimeParseError => e
+                  if @support_rfc5424_without_subseconds
+                    log.trace(e)
+                    @time_parser_rfc5424_without_subseconds.parse(time_str)
+                  else
+                    raise
+                  end
+                end
+              end
+              record[name] = value if @keep_time_key
+            else
+              record[name] = value
+            end
+          end
+        }
+
+        if @estimate_current_event
+          time ||= Fluent::EventTime.now
+        end
+
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_tsv.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class TSVParser < Parser
+      Plugin.register_parser('tsv', self)
+
+      desc 'Names of fields included in each lines'
+      config_param :keys, :array, value_type: :string
+      desc 'The delimiter character (or string) of TSV values'
+      config_param :delimiter, :string, default: "\t"
+
+      def configure(conf)
+        super
+        @key_num = @keys.length
+      end
+
+      def parse(text)
+        values = text.split(@delimiter, @key_num)
+        r = Hash[@keys.zip(values)]
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/socket_util.rb

@@ -14,149 +14,9 @@
 #    limitations under the License.
 #
 
-require 'ipaddr'
-
-require 'cool.io'
-
-require 'fluent/plugin'
+require 'fluent/compat/socket_util'
 
 module Fluent
-  module SocketUtil
-    def create_udp_socket(host)
-      if IPAddr.new(IPSocket.getaddress(host)).ipv4?
-        UDPSocket.new
-      else
-        UDPSocket.new(Socket::AF_INET6)
-      end
-    end
-    module_function :create_udp_socket
-
-    class UdpHandler < Coolio::IO
-      def initialize(io, log, body_size_limit, callback, resolve_hostname = false)
-        super(io)
-        @io = io
-        @io.do_not_reverse_lookup = !resolve_hostname
-        @log = log
-        @body_size_limit = body_size_limit
-        @callback = callback
-      end
-
-      def on_readable
-        msg, addr = @io.recvfrom_nonblock(@body_size_limit)
-        msg.chomp!
-        @callback.call(msg, addr)
-      rescue => e
-        @log.error "unexpected error", error: e, error_class: e.class
-      end
-    end
-
-    class TcpHandler < Coolio::Socket
-      PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
-
-      def initialize(io, log, delimiter, callback, resolve_hostname = false)
-        super(io)
-        if io.is_a?(TCPSocket)
-          io.do_not_reverse_lookup = resolve_hostname
-          @addr = (io.peeraddr rescue PEERADDR_FAILED)
-
-          opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-        end
-        @delimiter = delimiter
-        @callback = callback
-        @log = log
-        @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
-        @buffer = "".force_encoding('ASCII-8BIT')
-      end
-
-      def on_connect
-      end
-
-      def on_read(data)
-        @buffer << data
-        pos = 0
-
-        while i = @buffer.index(@delimiter, pos)
-          msg = @buffer[pos...i]
-          @callback.call(msg, @addr)
-          pos = i + @delimiter.length
-        end
-        @buffer.slice!(0, pos) if pos > 0
-      rescue => e
-        @log.error "unexpected error", error: e, error_class: e.class
-        close
-      end
-
-      def on_close
-        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
-      end
-    end
-
-    class BaseInput < Fluent::Input
-      def initialize
-        super
-        require 'fluent/parser'
-      end
-
-      desc 'Tag of output events.'
-      config_param :tag, :string
-      desc 'The format of the payload.'
-      config_param :format, :string
-      desc 'The port to listen to.'
-      config_param :port, :integer, default: 5150
-      desc 'The bind address to listen to.'
-      config_param :bind, :string, default: '0.0.0.0'
-      desc "Deprecated parameter. Use source_hostname_key instead"
-      config_param :source_host_key, :string, default: nil, deprecated: "Use source_hostname_key instead"
-      desc "The field name of the client's hostname."
-      config_param :source_hostname_key, :string, default: nil
-      config_param :blocking_timeout, :time, default: 0.5
-
-      def configure(conf)
-        super
-
-        @source_hostname_key = @source_host_key if @source_host_key
-        @parser = Plugin.new_parser(@format)
-        @parser.configure(conf)
-      end
-
-      def start
-        @loop = Coolio::Loop.new
-        @handler = listen(method(:on_message))
-        @loop.attach(@handler)
-        @thread = Thread.new(&method(:run))
-      end
-
-      def shutdown
-        @loop.watchers.each { |w| w.detach }
-        @loop.stop
-        @handler.close
-        @thread.join
-      end
-
-      def run
-        @loop.run(@blocking_timeout)
-      rescue => e
-        log.error "unexpected error", error: e, error_class: e.class
-        log.error_backtrace
-      end
-
-      private
-
-      def on_message(msg, addr)
-        @parser.parse(msg) { |time, record|
-          unless time && record
-            log.warn "pattern not match: #{msg.inspect}"
-            return
-          end
-
-          record[@source_hostname_key] = addr[2] if @source_hostname_key
-          router.emit(@tag, time, record)
-        }
-      rescue => e
-        log.error msg.dump, error: e, error_class: e.class, host: addr[3]
-        log.error_backtrace
-      end
-    end
-  end
+  # obsolete
+  SocketUtil = Fluent::Compat::SocketUtil
 end

data/lib/fluent/plugin/storage.rb

@@ -0,0 +1,84 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+
+module Fluent
+  module Plugin
+    class Storage < Base
+      include OwnedByMixin
+
+      DEFAULT_TYPE = 'local'
+
+      configured_in :storage
+
+      config_param :persistent,        :bool, default: false # load/save with all operations
+      config_param :autosave,          :bool, default: true
+      config_param :autosave_interval, :time, default: 10
+      config_param :save_at_shutdown,  :bool, default: true
+
+      def self.validate_key(key)
+        raise ArgumentError, "key must be a string (or symbol for to_s)" unless key.is_a?(String) || key.is_a?(Symbol)
+        key.to_s
+      end
+
+      attr_accessor :log
+
+      def persistent_always?
+        false
+      end
+
+      def synchronized?
+        false
+      end
+
+      def implementation
+        self
+      end
+
+      def load
+        # load storage data from any data source, or initialize storage internally
+      end
+
+      def save
+        # save internal data store into data source (to be loaded)
+      end
+
+      def get(key)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def fetch(key, defval)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def put(key, value)
+        # return value
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def delete(key)
+        # return deleted value
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def update(key, &block) # transactional get-and-update
+        raise NotImplementedError, "Implement this method in child class"
+      end
+    end
+  end
+end