fluentd 0.14.4-x86-mingw32

data/lib/fluent/compat/socket_util.rb

@@ -0,0 +1,165 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'ipaddr'
+
+require 'cool.io'
+
+require 'fluent/plugin'
+require 'fluent/input'
+
+module Fluent
+  module Compat
+    module SocketUtil
+      def create_udp_socket(host)
+        if IPAddr.new(IPSocket.getaddress(host)).ipv4?
+          UDPSocket.new
+        else
+          UDPSocket.new(Socket::AF_INET6)
+        end
+      end
+      module_function :create_udp_socket
+
+      class UdpHandler < Coolio::IO
+        def initialize(io, log, body_size_limit, callback)
+          super(io)
+          @io = io
+          @log = log
+          @body_size_limit = body_size_limit
+          @callback = callback
+        end
+
+        def on_readable
+          msg, addr = @io.recvfrom_nonblock(@body_size_limit)
+          msg.chomp!
+          @callback.call(msg, addr)
+        rescue => e
+          @log.error "unexpected error", error: e
+        end
+      end
+
+      class TcpHandler < Coolio::Socket
+        PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
+
+        def initialize(io, log, delimiter, callback)
+          super(io)
+          @timeout = 0
+          if io.is_a?(TCPSocket)
+            @addr = (io.peeraddr rescue PEERADDR_FAILED)
+
+            opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+            io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+          end
+          @delimiter = delimiter
+          @callback = callback
+          @log = log
+          @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
+          @buffer = "".force_encoding('ASCII-8BIT')
+        end
+
+        def on_connect
+        end
+
+        def on_read(data)
+          @buffer << data
+          pos = 0
+
+          while i = @buffer.index(@delimiter, pos)
+            msg = @buffer[pos...i]
+            @callback.call(msg, @addr)
+            pos = i + @delimiter.length
+          end
+          @buffer.slice!(0, pos) if pos > 0
+        rescue => e
+          @log.error "unexpected error", error: e
+          close
+        end
+
+        def on_close
+          @log.trace { "closed fluent socket object_id=#{self.object_id}" }
+        end
+      end
+
+      class BaseInput < Fluent::Input
+        def initialize
+          super
+          require 'fluent/parser'
+        end
+
+        desc 'Tag of output events.'
+        config_param :tag, :string
+        desc 'The format of the payload.'
+        config_param :format, :string
+        desc 'The port to listen to.'
+        config_param :port, :integer, default: 5150
+        desc 'The bind address to listen to.'
+        config_param :bind, :string, default: '0.0.0.0'
+        desc "The field name of the client's hostname."
+        config_param :source_host_key, :string, default: nil
+        config_param :blocking_timeout, :time, default: 0.5
+
+        def configure(conf)
+          super
+
+          @parser = Plugin.new_parser(@format)
+          @parser.configure(conf)
+        end
+
+        def start
+          super
+
+          @loop = Coolio::Loop.new
+          @handler = listen(method(:on_message))
+          @loop.attach(@handler)
+          @thread = Thread.new(&method(:run))
+        end
+
+        def shutdown
+          @loop.watchers.each { |w| w.detach }
+          @loop.stop if @loop.instance_variable_get("@running")
+          @handler.close
+          @thread.join
+
+          super
+        end
+
+        def run
+          @loop.run(@blocking_timeout)
+        rescue => e
+          log.error "unexpected error", error: e
+          log.error_backtrace
+        end
+
+        private
+
+        def on_message(msg, addr)
+          @parser.parse(msg) { |time, record|
+            unless time && record
+              log.warn "pattern not match: #{msg.inspect}"
+              return
+            end
+
+            record[@source_host_key] = addr[3] if @source_host_key
+            router.emit(@tag, time, record)
+          }
+        rescue => e
+          log.error msg.dump, error: e, host: addr[3]
+          log.error_backtrace
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/string_util.rb

@@ -0,0 +1,34 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module StringUtil
+      def match_regexp(regexp, string)
+        begin
+          return regexp.match(string)
+        rescue ArgumentError => e
+          raise e unless e.message.index("invalid byte sequence in".freeze).zero?
+          $log.info "invalid byte sequence is replaced in `#{string}`"
+          string = string.scrub('?')
+          retry
+        end
+        return true
+      end
+      module_function :match_regexp
+    end
+  end
+end

data/lib/fluent/compat/structured_format_mixin.rb

@@ -0,0 +1,26 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module StructuredFormatMixin
+      def format(tag, time, record)
+        filter_record(tag, time, record)
+        format_record(record)
+      end
+    end
+  end
+end

data/lib/fluent/compat/type_converter.rb

@@ -0,0 +1,90 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module TypeConverter
+      Converters = {
+        'string' => lambda { |v| v.to_s },
+        'integer' => lambda { |v| v.to_i },
+        'float' => lambda { |v| v.to_f },
+        'bool' => lambda { |v|
+          case v.downcase
+          when 'true', 'yes', '1'
+            true
+          else
+            false
+          end
+        },
+        'time' => lambda { |v, time_parser|
+          time_parser.parse(v)
+        },
+        'array' => lambda { |v, delimiter|
+          v.to_s.split(delimiter)
+        }
+      }
+
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :types, :string, default: nil
+          config_param :types_delimiter, :string, default: ','
+          config_param :types_label_delimiter, :string, default: ':'
+        }
+      end
+
+      def configure(conf)
+        super
+
+        @type_converters = nil
+        @type_converters = parse_types_parameter unless @types.nil?
+      end
+
+      private
+
+      def convert_type(name, value)
+        converter = @type_converters[name]
+        converter.nil? ? value : converter.call(value)
+      end
+
+      def parse_types_parameter
+        converters = {}
+
+        @types.split(@types_delimiter).each { |pattern_name|
+          name, type, format = pattern_name.split(@types_label_delimiter, 3)
+          raise ConfigError, "Type is needed" if type.nil?
+
+          case type
+          when 'time'
+            require 'fluent/parser'
+            t_parser = Fluent::TextParser::TimeParser.new(format)
+            converters[name] = lambda { |v|
+              Converters[type].call(v, t_parser)
+            }
+          when 'array'
+            delimiter = format || ','
+            converters[name] = lambda { |v|
+              Converters[type].call(v, delimiter)
+            }
+          else
+            converters[name] = Converters[type]
+          end
+        }
+
+        converters
+      end
+    end
+  end
+end

data/lib/fluent/config.rb

@@ -0,0 +1,56 @@
+#
+# Fluent
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+require 'fluent/config/element'
+require 'fluent/configurable'
+
+module Fluent
+  module Config
+    def self.parse(str, fname, basepath = Dir.pwd, v1_config = nil, syntax: :v1)
+      parser = if fname =~ /\.rb$/ || syntax == :ruby
+                 :ruby
+               elsif v1_config.nil?
+                 case syntax
+                 when :v1 then :v1
+                 when :v0 then :v0
+                 else
+                   raise ArgumentError, "Unknown Fluentd configuration syntax: '#{syntax}'"
+                 end
+               elsif v1_config then :v1
+               else :v0
+               end
+      case parser
+      when :v1
+        require 'fluent/config/v1_parser'
+        V1Parser.parse(str, fname, basepath, Kernel.binding)
+      when :v0
+        # TODO: show deprecated message in v1
+        require 'fluent/config/parser'
+        Parser.parse(str, fname, basepath)
+      when :ruby
+        require 'fluent/config/dsl'
+        Config::DSL::Parser.parse(str, File.join(basepath, fname))
+      else
+        raise "[BUG] unknown configuration parser specification:'#{parser}'"
+      end
+    end
+
+    def self.new(name = '')
+      Element.new(name, '', {}, [])
+    end
+  end
+end

data/lib/fluent/config/basic_parser.rb

@@ -0,0 +1,123 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'stringio'
+require 'fluent/config/error'
+
+module Fluent
+  module Config
+    class BasicParser
+      def initialize(strscan)
+        @ss = strscan
+      end
+
+      LINE_END = /(?:[ \t]*(?:\#.*)?(?:\z|[\r\n]))+/
+      SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
+      ZERO_OR_MORE_SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))*/
+      SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/
+      LINE_END_WITHOUT_SPACING_AND_COMMENT = /(?:\z|[\r\n])/
+
+      module ClassMethods
+        def symbol(string)
+          /#{Regexp.escape(string)}/
+        end
+
+        def def_symbol(method_name, string)
+          pattern = symbol(string)
+          define_method(method_name) do
+            skip(pattern) && string
+          end
+        end
+
+        def def_literal(method_name, string)
+          pattern = /#{string}#{LINE_END}/
+          define_method(method_name) do
+            skip(pattern) && string
+          end
+        end
+      end
+
+      extend ClassMethods
+
+      def skip(pattern)
+        @ss.skip(pattern)
+      end
+
+      def scan(pattern)
+        @ss.scan(pattern)
+      end
+
+      def getch
+        @ss.getch
+      end
+
+      def eof?
+        @ss.eos?
+      end
+
+      def prev_match
+        @ss[0]
+      end
+
+      def check(pattern)
+        @ss.check(pattern)
+      end
+
+      def line_end
+        skip(LINE_END)
+      end
+
+      def spacing
+        skip(SPACING)
+      end
+
+      def spacing_without_comment
+        skip(SPACING_WITHOUT_COMMENT)
+      end
+
+      def parse_error!(message)
+        raise ConfigParseError, "#{message} at #{error_sample}"
+      end
+
+      def error_sample
+        pos = @ss.pos
+
+        lines = @ss.string.lines.to_a
+        lines.each_with_index { |line, ln|
+          if line.size >= pos
+            msgs = ["line #{ln + 1},#{pos}\n"]
+
+            if ln > 0
+              last_line = lines[ln - 1]
+              msgs << "%3s: %s" % [ln, last_line]
+            end
+
+            msgs << "%3s: %s" % [ln + 1, line]
+            msgs << "\n     #{'-' * pos}^\n"
+
+            if next_line = lines[ln + 1]
+              msgs << "%3s: %s" % [ln + 2, next_line]
+            end
+
+            return msgs.join
+          end
+          pos -= line.size
+          last_line = line
+        }
+      end
+    end
+  end
+end