fluentd 0.12.0.pre.1 → 0.12.0.pre.2

data/lib/fluent/plugin/filter_grep.rb

@@ -0,0 +1,69 @@
+module Fluent
+  class GrepFilter < Filter
+    Fluent::Plugin.register_filter('grep', self)
+
+    REGEXP_MAX_NUM = 20
+
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, :default => nil }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, :default => nil }
+
+    # for test
+    attr_reader :regexps
+    attr_reader :excludes
+
+    def configure(conf)
+      super
+
+      @regexps = {}
+      (1..REGEXP_MAX_NUM).each do |i|
+        next unless conf["regexp#{i}"]
+        key, regexp = conf["regexp#{i}"].split(/ /, 2)
+        raise ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
+        raise ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
+        @regexps[key] = Regexp.compile(regexp)
+      end
+
+      @excludes = {}
+      (1..REGEXP_MAX_NUM).each do |i|
+        next unless conf["exclude#{i}"]
+        key, exclude = conf["exclude#{i}"].split(/ /, 2)
+        raise ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
+        raise ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
+        @excludes[key] = Regexp.compile(exclude)
+      end
+    end
+
+    def filter_stream(tag, es)
+      result_es = MultiEventStream.new
+      es.each do |time, record|
+        catch(:break_loop) do
+          @regexps.each do |key, regexp|
+            throw :break_loop unless match(regexp, record[key].to_s)
+          end
+          @excludes.each do |key, exclude|
+            throw :break_loop if match(exclude, record[key].to_s)
+          end
+          result_es.add(time, record)
+        end
+      end
+      result_es
+    rescue => e
+      log.warn "failed to grep events", :error_class => e.class, :error => e.message
+      log.warn_backtrace
+    end
+
+    private
+
+    def match(regexp, string)
+      begin
+        return regexp.match(string)
+      rescue ArgumentError => e
+        raise e unless e.message.index("invalid byte sequence in".freeze).zero?
+        log.info "invalid byte sequence is replaced in `#{string}`"
+        string = string.scrub('?')
+        retry
+      end
+      return true
+    end
+  end
+end

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -0,0 +1,183 @@
+require 'ostruct'
+
+module Fluent
+  class RecordTransformerFilter < Filter
+    Fluent::Plugin.register_filter('record_transformer', self)
+
+    def initialize
+      require 'socket'
+      super
+    end
+
+    config_param :remove_keys, :string, :default => nil
+    config_param :keep_keys, :string, :default => nil
+    config_param :renew_record, :bool, :default => false
+    config_param :enable_ruby, :bool, :default => false
+
+    def configure(conf)
+      super
+
+      @map = {}
+      # <record></record> directive
+      conf.elements.select { |element| element.name == 'record' }.each do |element|
+        element.each_pair do |k, v|
+          element.has_key?(k) # to suppress unread configuration warning
+          @map[k] = v
+        end
+      end
+
+      if @remove_keys
+        @remove_keys = @remove_keys.split(',')
+      end
+
+      if @keep_keys
+        raise Fluent::ConfigError, "`renew_record` must be true to use `keep_keys`" unless @renew_record
+        @keep_keys = @keep_keys.split(',')
+      end
+
+      @placeholder_expander =
+        if @enable_ruby
+          # require utilities which would be used in ruby placeholders
+          require 'pathname'
+          require 'uri'
+          require 'cgi'
+          RubyPlaceholderExpander.new(log)
+        else
+          PlaceholderExpander.new(log)
+        end
+
+      @hostname = Socket.gethostname
+    end
+
+    def filter_stream(tag, es)
+      new_es = MultiEventStream.new
+      tag_parts = tag.split('.')
+      tag_prefix = tag_prefix(tag_parts)
+      tag_suffix = tag_suffix(tag_parts)
+      placeholders = {
+        'tag' => tag,
+        'tag_parts' => tag_parts,
+        'tag_prefix' => tag_prefix,
+        'tag_suffix' => tag_suffix,
+        'hostname' => @hostname,
+      }
+      last_record = nil
+      es.each do |time, record|
+        last_record = record # for debug log
+        new_record = reform(time, record, placeholders)
+        new_es.add(time, new_record)
+      end
+      new_es
+    rescue => e
+      log.warn "failed to reform records", :error_class => e.class, :error => e.message
+      log.warn_backtrace
+      log.debug "map:#{@map} record:#{last_record} placeholders:#{placeholders}"
+    end
+
+    private
+
+    def reform(time, record, opts)
+      @placeholder_expander.prepare_placeholders(time, record, opts)
+
+      new_record = @renew_record ? {} : record.dup
+      @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
+      @map.each_pair {|k, v| new_record[k] = @placeholder_expander.expand(v) }
+      @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
+
+      new_record
+    end
+
+    def tag_prefix(tag_parts)
+      return [] if tag_parts.empty?
+      tag_prefix = [tag_parts.first]
+      1.upto(tag_parts.size-1).each do |i|
+        tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}"
+      end
+      tag_prefix
+    end
+
+    def tag_suffix(tag_parts)
+      return [] if tag_parts.empty?
+      rev_tag_parts = tag_parts.reverse
+      rev_tag_suffix = [rev_tag_parts.first]
+      1.upto(tag_parts.size-1).each do |i|
+        rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}"
+      end
+      rev_tag_suffix.reverse!
+    end
+
+    class PlaceholderExpander
+      attr_reader :placeholders, :log
+
+      def initialize(log)
+        @log = log
+      end
+
+      def prepare_placeholders(time, record, opts)
+        placeholders = { '${time}' => Time.at(time).to_s }
+        record.each {|key, value| placeholders.store("${#{key}}", value) }
+
+        opts.each do |key, value|
+          if value.kind_of?(Array) # tag_parts, etc
+            size = value.size
+            value.each_with_index { |v, idx|
+              placeholders.store("${#{key}[#{idx}]}", v)
+              placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1]
+            }
+          else # string, interger, float, and others?
+            placeholders.store("${#{key}}", value)
+          end
+        end
+
+        @placeholders = placeholders
+      end
+
+      def expand(str)
+        str.gsub(/(\${[a-zA-Z0-9_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
+          log.warn "unknown placeholder `#{$1}` found" unless @placeholders.include?($1)
+          @placeholders[$1]
+        }
+      end
+    end
+
+    class RubyPlaceholderExpander
+      attr_reader :placeholders, :log
+
+      def initialize(log)
+        @log = log
+      end
+
+      # Get placeholders as a struct
+      #
+      # @param [Time]   time        the time
+      # @param [Hash]   record      the record
+      # @param [Hash]   opts        others
+      def prepare_placeholders(time, record, opts)
+        struct = UndefOpenStruct.new(record)
+        struct.time = Time.at(time)
+        opts.each {|key, value| struct.__send__("#{key}=", value) }
+        @placeholders = struct
+      end
+
+      # Replace placeholders in a string
+      #
+      # @param [String] str         the string to be replaced
+      def expand(str)
+        interpolated = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
+        begin
+          eval "\"#{interpolated}\"", @placeholders.instance_eval { binding }
+        rescue => e
+          log.warn "failed to expand `#{str}`", :error_class => e.class, :error => e.message
+          log.warn_backtrace
+          nil
+        end
+      end
+
+      class UndefOpenStruct < OpenStruct
+        (Object.instance_methods).each do |m|
+          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_exec.rb

@@ -21,6 +21,7 @@ module Fluent
     def initialize
       super
       require 'fluent/plugin/exec_util'
+      require 'fluent/timezone'
     end
 
     SUPPORTED_FORMAT = {
@@ -53,6 +54,11 @@ module Fluent
         @localtime = false
       end
 
+      if conf['timezone']
+        @timezone = conf['timezone']
+        Fluent::Timezone.validate!(@timezone)
+      end
+
       if !@tag && !@tag_key
         raise ConfigError, "'tag' or 'tag_key' option is required on exec input"
       end

data/lib/fluent/plugin/in_forward.rb

@@ -54,7 +54,16 @@ module Fluent
     end
 
     def shutdown
-      @loop.watchers.each {|w| w.detach }
+      # In test cases it occasionally appeared that when detaching a watcher, another watcher is also detached.
+      # In the case in the iteration of watchers, a watcher that has been already detached is intended to be detached
+      # and therfore RuntimeError occurs saying that it is not attached to a loop.
+      # It occures only when testing for sending responses to ForwardOutput.
+      # Sending responses needs to write the socket that is previously used only to read
+      # and a handler has 2 watchers that is used to read and to write.
+      # This problem occurs possibly because those watchers are thought to be related to each other
+      # and when detaching one of them the other is also detached for some reasons.
+      # As a workaround, check if watchers are attached before detaching them.
+      @loop.watchers.each {|w| w.detach if w.attached? }
       @loop.stop
       @usock.close
       @thread.join
@@ -95,17 +104,20 @@ module Fluent
     # message Forward {
     #   1: string tag
     #   2: list<Entry> entries
+    #   3: object option (optional)
     # }
     #
     # message PackedForward {
     #   1: string tag
     #   2: raw entries  # msgpack stream of Entry
+    #   3: object option (optional)
     # }
     #
     # message Message {
     #   1: string tag
     #   2: long? time
     #   3: object record
+    #   4: object option (optional)
     # }
     def on_message(msg, chunk_size, source)
       if msg.nil?
@@ -128,6 +140,7 @@ module Fluent
         # PackedForward
         es = MessagePackEventStream.new(entries)
         router.emit_stream(tag, es)
+        option = msg[2]
 
       elsif entries.class == Array
         # Forward
@@ -140,6 +153,7 @@ module Fluent
           es.add(time, record)
         }
         router.emit_stream(tag, es)
+        option = msg[2]
 
       else
         # Message
@@ -148,7 +162,11 @@ module Fluent
         time = msg[1]
         time = Engine.now if time == 0
         router.emit(tag, time, record)
+        option = msg[3]
       end
+
+      # return option for response
+      option
     end
 
     class Handler < Coolio::Socket
@@ -186,13 +204,16 @@ module Fluent
         first = data[0]
         if first == '{' || first == '['
           m = method(:on_read_json)
+          @serializer = :to_json.to_proc
           @y = Yajl::Parser.new
           @y.on_parse_complete = lambda { |obj|
-            @on_message.call(obj, @chunk_counter, @source)
+            option = @on_message.call(obj, @chunk_counter, @source)
+            respond option if option
             @chunk_counter = 0
           }
         else
           m = method(:on_read_msgpack)
+          @serializer = :to_msgpack.to_proc
           @u = MessagePack::Unpacker.new
         end
 
@@ -214,7 +235,8 @@ module Fluent
       def on_read_msgpack(data)
         @chunk_counter += data.bytesize
         @u.feed_each(data) do |obj|
-          @on_message.call(obj, @chunk_counter, @source)
+          option = @on_message.call(obj, @chunk_counter, @source)
+          respond option if option
           @chunk_counter = 0
         end
       rescue => e
@@ -223,8 +245,16 @@ module Fluent
         close
       end
 
+      def respond(option)
+        if option && option['chunk']
+          res = { 'ack' => option['chunk'] }
+          write @serializer.call(res)
+          @log.trace { "sent response to fluent socket" }
+        end
+      end
+
       def on_close
-        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
+        @log.trace { "closed socket" }
       end
     end
 

data/lib/fluent/plugin/in_http.rb

@@ -190,7 +190,6 @@ module Fluent
         @km = km
         @callback = callback
         @body_size_limit = body_size_limit
-        @content_type = ""
         @next_close = false
         @format = format
         @log = log
@@ -235,6 +234,7 @@ module Fluent
           @keep_alive = false
         end
         @env = {}
+        @content_type = ""
         headers.each_pair {|k,v|
           @env["HTTP_#{k.gsub('-','_').upcase}"] = v
           case k

data/lib/fluent/plugin/out_exec.rb

@@ -56,7 +56,7 @@ module Fluent
 
       if @time_key
         if @time_format
-          tf = TimeFormatter.new(@time_format, @localtime)
+          tf = TimeFormatter.new(@time_format, @localtime, @timezone)
           @time_format_proc = tf.method(:format)
         else
           @time_format_proc = Proc.new { |time| time.to_s }

data/lib/fluent/plugin/out_exec_filter.rb

@@ -22,6 +22,7 @@ module Fluent
 
     def initialize
       super
+      require 'fluent/timezone'
     end
 
     config_param :command, :string
@@ -59,6 +60,7 @@ module Fluent
     config_param :time_format, :string, :default => nil
 
     config_param :localtime, :bool, :default => true
+    config_param :timezone, :string, :default => nil
     config_param :num_children, :integer, :default => 1
 
     # nil, 'none' or 0: no respawn, 'inf' or -1: infinite times, positive integer: try to respawn specified times only
@@ -96,13 +98,18 @@ module Fluent
         @localtime = false
       end
 
+      if conf['timezone']
+        @timezone = conf['timezone']
+        Fluent::Timezone.validate!(@timezone)
+      end
+
       if !@tag && !@out_tag_key
         raise ConfigError, "'tag' or 'out_tag_key' option is required on exec_filter output"
       end
 
       if @in_time_key
         if f = @in_time_format
-          tf = TimeFormatter.new(f, @localtime)
+          tf = TimeFormatter.new(f, @localtime, @timezone)
           @time_format_proc = tf.method(:format)
         else
           @time_format_proc = Proc.new {|time| time.to_s }

data/lib/fluent/plugin/out_forward.rb

@@ -15,11 +15,21 @@
 #
 
 module Fluent
+  class ForwardOutputError < StandardError
+  end
+
+  class ForwardOutputResponseError < ForwardOutputError
+  end
+
+  class ForwardOutputACKTimeoutError < ForwardOutputResponseError
+  end
+
   class ForwardOutput < ObjectBufferedOutput
     Plugin.register_output('forward', self)
 
     def initialize
       super
+      require "base64"
       require 'socket'
       require 'fileutils'
       require 'fluent/plugin/socket_util'
@@ -43,12 +53,22 @@ module Fluent
     config_param :expire_dns_cache, :time, :default => nil  # 0 means disable cache
     config_param :phi_threshold, :integer, :default => 16
     config_param :phi_failure_detector, :bool, :default => true
+
+    # if any options added that requires extended forward api, fix @extend_internal_protocol
+
+    config_param :require_ack_response, :bool, :default => false  # require in_forward to respond with ack
+    config_param :ack_response_timeout, :time, :default => 190  # 0 means do not wait for ack responses
+    # Linux default tcp_syn_retries is 5 (in many environment)
+    # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
+
     attr_reader :nodes
 
     # backward compatibility
     config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
     config_param :host, :string, :default => nil
 
+    attr_accessor :extend_internal_protocol
+
     def configure(conf)
       super
 
@@ -64,6 +84,12 @@ module Fluent
 
       recover_sample_size = @recover_wait / @heartbeat_interval
 
+      # add options here if any options addes which uses extended protocol
+      @extend_internal_protocol = if @require_ack_response
+                                    true
+                                  else
+                                    false
+                                  end
       conf.elements.each {|e|
         next if e.name != "server"
 
@@ -86,7 +112,7 @@ module Fluent
         node_conf = NodeConfig.new(name, host, port, weight, standby, failure,
           @phi_threshold, recover_sample_size, @expire_dns_cache, @phi_failure_detector)
         @nodes << Node.new(log, node_conf)
-        log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight
+        log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight, :plugin_id=>plugin_id
       }
     end
 
@@ -200,8 +226,17 @@ module Fluent
       @weight_array = weight_array
     end
 
-    # MessagePack FixArray length = 2
-    FORWARD_HEADER = [0x92].pack('C')
+    # MessagePack FixArray length = 3 (if @extend_internal_protocol)
+    #                             = 2 (else)
+    FORWARD_HEADER = [0x92].pack('C').freeze
+    FORWARD_HEADER_EXT = [0x93].pack('C').freeze
+    def forward_header
+      if @extend_internal_protocol
+        FORWARD_HEADER_EXT
+      else
+        FORWARD_HEADER
+      end
+    end
 
     #FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
     def send_heartbeat_tcp(node)
@@ -229,7 +264,7 @@ module Fluent
         sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
 
         # beginArray(2)
-        sock.write FORWARD_HEADER
+        sock.write forward_header
 
         # writeRaw(tag)
         sock.write tag.to_msgpack  # tag
@@ -250,7 +285,46 @@ module Fluent
         # writeRawBody(packed_es)
         chunk.write_to(sock)
 
+        if @extend_internal_protocol
+          option = {}
+          option['chunk'] = Base64.encode64(chunk.unique_id) if @require_ack_response
+          sock.write option.to_msgpack
+
+          if @require_ack_response && @ack_response_timeout > 0
+            # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
+            # To avoid a decrease of troughput, it is necessary to prepare a list of chunks that wait for responses
+            # and process them asynchronously.
+            if IO.select([sock], nil, nil, @ack_response_timeout)
+              raw_data = sock.recv(1024)
+
+              # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+              # In this case, the node is available and successfully close connection without sending responses.
+              # ForwardInput is not expected to do so, but some alternatives may do so.
+              # Therefore do not send the chunk again.
+              unless raw_data.empty?
+                # Serialization type of the response is same as sent data.
+                res = MessagePack.unpack(raw_data)
+
+                if res['ack'] != option['chunk']
+                  # Some errors may have occured when ack and chunk id is different, so send the chunk again.
+                  raise ForwardOutputResponseError, "ack in response and chunk id in sent data are different"
+                end
+              end
+
+            else
+              # IO.select returns nil on timeout.
+              # There are 2 types of cases when no response has been received:
+              # (1) the node does not support sending responses
+              # (2) the node does support sending response but responses have not arrived for some reasons.
+              @log.warn "no response from #{node.host}:#{node.port}. regard it as unavailable."
+              node.disable!
+              raise ForwardOutputACKTimeoutError, "node #{node.host}:#{node.port} does not return ACK"
+            end
+          end
+        end
+
         node.heartbeat(false)
+        return res  # for test
       ensure
         sock.close
       end
@@ -354,6 +428,10 @@ module Fluent
         @available
       end
 
+      def disable!
+        @available = false
+      end
+
       def standby?
         @conf.standby
       end