fluentd 1.10.1-x86-mingw32 → 1.11.1-x86-mingw32

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -223,7 +223,7 @@ module Fluent::Plugin
         opts = {with_config: false, with_retry: false}
         timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
           es = Fluent::MultiEventStream.new
-          now = Fluent::Engine.now
+          now = Fluent::EventTime.now
           plugins_info_all(opts).each { |record|
             es.add(now, record)
           }

data/lib/fluent/plugin/in_syslog.rb

@@ -181,12 +181,12 @@ module Fluent::Plugin
           if octet_count_frame
             while idx = buffer.index(delimiter, pos)
               num = Integer(buffer[pos..idx])
-              pos = idx + num
-              msg = buffer[idx + 1...pos]
-              if msg.size < num - 1
-                pos = pos - num - num.to_s.size
+              msg = buffer[idx + delimiter_size, num]
+              if msg.size != num
                 break
               end
+
+              pos = idx + delimiter_size + num
               message_handler(msg, conn)
             end
           else

data/lib/fluent/plugin/in_tail.rb

@@ -136,7 +136,7 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
       end
 
-      @paths = @path.split(@path_delimiter).map(&:strip)
+      @paths = @path.split(@path_delimiter).map(&:strip).uniq
       if @paths.empty?
         raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
       end
@@ -271,9 +271,9 @@ module Fluent::Plugin
                 end
               else
                 if is_file
-                  unless @ignore_list.include?(path)
+                  unless @ignore_list.include?(p)
                     log.warn "#{p} unreadable. It is excluded and would be examined next time."
-                    @ignore_list << path if @ignore_repeated_permission_error
+                    @ignore_list << p if @ignore_repeated_permission_error
                   end
                 end
                 false
@@ -296,7 +296,7 @@ module Fluent::Plugin
                end
         path.include?('*') ? Dir.glob(path) : path
       }.flatten.uniq
-      paths - excluded
+      paths.uniq - excluded
     end
 
     # in_tail with '*' path doesn't check rotation file equality at refresh phase.

data/lib/fluent/plugin/in_unix.rb

@@ -14,50 +14,68 @@
 #    limitations under the License.
 #
 
-require 'fileutils'
-require 'socket'
+require 'fluent/env'
+require 'fluent/plugin/input'
+require 'fluent/msgpack_factory'
 
 require 'cool.io'
 require 'yajl'
+require 'fileutils'
+require 'socket'
+
+module Fluent::Plugin
+  # TODO: This plugin will be 3rd party plugin
+  class UnixInput < Input
+    Fluent::Plugin.register_input('unix', self)
+
+    helpers :event_loop
+
+    def initialize
+      super
 
-require 'fluent/input'
-require 'fluent/event'
+      @lsock = nil
+    end
 
-module Fluent
-  # obsolete
-  class StreamInput < Input
-    config_param :blocking_timeout, :time, default: 0.5
+    desc 'The path to your Unix Domain Socket.'
+    config_param :path, :string, default: Fluent::DEFAULT_SOCKET_PATH
+    desc 'The backlog of Unix Domain Socket.'
+    config_param :backlog, :integer, default: nil
+    desc "New tag instead of incoming tag"
+    config_param :tag, :string, default: nil
+
+    def configure(conf)
+      super
+    end
 
     def start
       super
 
-      @loop = Coolio::Loop.new
       @lsock = listen
-      @loop.attach(@lsock)
-      @thread = Thread.new(&method(:run))
+      event_loop_attach(@lsock)
     end
 
     def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @lsock.close
-      @thread.join
+      if @lsock
+        event_loop_detach(@lsock)
+        @lsock.close
+      end
 
       super
     end
 
-    #def listen
-    #end
+    def listen
+      if File.exist?(@path)
+        log.warn "Found existing '#{@path}'. Remove this file for in_unix plugin"
+        File.unlink(@path)
+      end
+      FileUtils.mkdir_p(File.dirname(@path))
 
-    def run
-      @loop.run(@blocking_timeout)
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
+      log.info "listening fluent socket on #{@path}"
+      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
+      s.listen(@backlog) unless @backlog.nil?
+      s
     end
 
-    private
-
     # message Entry {
     #   1: long time
     #   2: object record
@@ -79,23 +97,27 @@ module Fluent
     #   3: object record
     # }
     def on_message(msg)
-      # TODO format error
-      tag = msg[0].to_s
+      unless msg.is_a?(Array)
+        log.warn "incoming data is broken:", msg: msg
+        return
+      end
+
+      tag = @tag || (msg[0].to_s)
       entries = msg[1]
 
-      if entries.class == String
+      case entries
+      when String
         # PackedForward
-        es = MessagePackEventStream.new(entries)
+        es = Fluent::MessagePackEventStream.new(entries)
         router.emit_stream(tag, es)
 
-      elsif entries.class == Array
+      when Array
         # Forward
-        es = MultiEventStream.new
+        es = Fluent::MultiEventStream.new
         entries.each {|e|
           record = e[1]
           next if record.nil?
-          time = e[0]
-          time = (now ||= Engine.now) if time.to_i == 0
+          time = convert_time(e[0])
           es.add(time, record)
         }
         router.emit_stream(tag, es)
@@ -105,25 +127,28 @@ module Fluent
         record = msg[2]
         return if record.nil?
 
-        time = msg[1]
-        time = Engine.now if time.to_i == 0
+        time = convert_time(msg[1])
         router.emit(tag, time, record)
       end
     end
 
+    def convert_time(time)
+      case time
+      when nil, 0
+        Fluent::EventTime.now
+      when Fluent::EventTime
+        time
+      else
+        Fluent::EventTime.from_time(Time.at(time))
+      end
+    end
+
     class Handler < Coolio::Socket
       def initialize(io, log, on_message)
         super(io)
-        if io.is_a?(TCPSocket)
-          opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-        end
+
         @on_message = on_message
         @log = log
-        @log.trace {
-          remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername) rescue nil
-          "accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
-        }
       end
 
       def on_connect
@@ -131,13 +156,13 @@ module Fluent
 
       def on_read(data)
         first = data[0]
-        if first == '{' || first == '['
+        if first == '{'.freeze || first == '['.freeze
           m = method(:on_read_json)
-          @y = Yajl::Parser.new
-          @y.on_parse_complete = @on_message
+          @parser = Yajl::Parser.new
+          @parser.on_parse_complete = @on_message
         else
           m = method(:on_read_msgpack)
-          @u = Fluent::MessagePackFactory.msgpack_unpacker
+          @parser = Fluent::MessagePackFactory.msgpack_unpacker
         end
 
         singleton_class.module_eval do
@@ -147,17 +172,17 @@ module Fluent
       end
 
       def on_read_json(data)
-        @y << data
-      rescue
-        @log.error "unexpected error", error: $!.to_s
+        @parser << data
+      rescue => e
+        @log.error "unexpected error in json payload", error: e.to_s
         @log.error_backtrace
         close
       end
 
       def on_read_msgpack(data)
-        @u.feed_each(data, &@on_message)
-      rescue
-        @log.error "unexpected error", error: $!.to_s
+        @parser.feed_each(data, &@on_message)
+      rescue => e
+        @log.error "unexpected error in msgpack payload", error: e.to_s
         @log.error_backtrace
         close
       end
@@ -167,29 +192,4 @@ module Fluent
       end
     end
   end
-
-  class UnixInput < StreamInput
-    Plugin.register_input('unix', self)
-
-    desc 'The path to your Unix Domain Socket.'
-    config_param :path, :string, default: DEFAULT_SOCKET_PATH
-    desc 'The backlog of Unix Domain Socket.'
-    config_param :backlog, :integer, default: nil
-
-    def configure(conf)
-      super
-      #log.warn "'unix' input is obsoleted and will be removed. Use 'forward' instead."
-    end
-
-    def listen
-      if File.exist?(@path)
-        File.unlink(@path)
-      end
-      FileUtils.mkdir_p File.dirname(@path)
-      log.info "listening fluent socket on #{@path}"
-      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
-      s.listen(@backlog) unless @backlog.nil?
-      s
-    end
-  end
 end

data/lib/fluent/plugin/out_copy.rb

@@ -61,7 +61,7 @@ module Fluent::Plugin
           output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
         rescue => e
           if @ignore_errors[i]
-            log.error "ignore emit error", error: e
+            log.error "ignore emit error in #{output.plugin_id}", error: e
           else
             raise e
           end

data/lib/fluent/plugin/out_file.rb

@@ -155,7 +155,7 @@ module Fluent::Plugin
         dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
         dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
 
-        test_chunk1 = chunk_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
+        test_chunk1 = chunk_for_test(dummy_tag, Fluent::EventTime.now, dummy_record)
         test_path = extract_placeholders(@path_template, test_chunk1)
         unless ::Fluent::FileUtil.writable_p?(test_path)
           raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"

data/lib/fluent/plugin/out_forward.rb

@@ -324,7 +324,7 @@ module Fluent::Plugin
         end
       end
 
-      if @keepalive && @keepalive_timeout
+      if @keepalive
         timer_execute(:out_forward_keep_alived_socket_watcher, @keep_alive_watcher_interval, &method(:on_purge_obsolete_socks))
       end
     end
@@ -382,7 +382,12 @@ module Fluent::Plugin
           cert_logical_store_name: @tls_cert_logical_store_name,
           cert_use_enterprise_store: @tls_cert_use_enterprise_store,
 
-          linger_timeout: @send_timeout,
+          # Enabling SO_LINGER causes tcp port exhaustion on Windows.
+          # This is because dynamic ports are only 16384 (from 49152 to 65535) and
+          # expiring SO_LINGER enabled ports should wait 4 minutes
+          # where set by TcpTimeDelay. Its default value is 4 minutes.
+          # So, we should disable SO_LINGER on Windows to prevent flood of waiting ports.
+          linger_timeout: Fluent.windows? ? nil : @send_timeout,
           send_timeout: @send_timeout,
           recv_timeout: @ack_response_timeout,
           connect_timeout: @connect_timeout,
@@ -578,12 +583,7 @@ module Fluent::Plugin
 
       def verify_connection
         connect do |sock, ri|
-          if ri.state != :established
-            establish_connection(sock, ri)
-            if ri.state != :established
-              raise "Failed to establish connection to #{@host}:#{@port}"
-            end
-          end
+          ensure_established_connection(sock, ri)
         end
       end
 
@@ -652,14 +652,7 @@ module Fluent::Plugin
       def send_data(tag, chunk)
         ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self)
         connect(nil, ack: ack) do |sock, ri|
-          if ri.state != :established
-            establish_connection(sock, ri)
-
-            if ri.state != :established
-              raise ConnectionClosedError, "failed to establish connection with node #{@name}"
-            end
-          end
-
+          ensure_established_connection(sock, ri)
           send_data_actual(sock, tag, chunk)
         end
 
@@ -684,7 +677,9 @@ module Fluent::Plugin
 
         case @sender.heartbeat_type
         when :transport
-          connect(dest_addr) do |_ri, _sock|
+          connect(dest_addr) do |sock, ri|
+            ensure_established_connection(sock, ri)
+
             ## don't send any data to not cause a compatibility problem
             # sock.write FORWARD_TCP_HEARTBEAT_DATA
 
@@ -714,7 +709,7 @@ module Fluent::Plugin
           @resolved_host ||= resolve_dns!
 
         else
-          now = Fluent::Engine.now
+          now = Fluent::EventTime.now
           rh = @resolved_host
           if !rh || now - @resolved_time >= @sender.expire_dns_cache
             rh = @resolved_host = resolve_dns!
@@ -776,6 +771,16 @@ module Fluent::Plugin
 
       private
 
+      def ensure_established_connection(sock, request_info)
+        if request_info.state != :established
+          establish_connection(sock, request_info)
+
+          if request_info.state != :established
+            raise ConnectionClosedError, "failed to establish connection with node #{@name}"
+          end
+        end
+      end
+
       def connect(host = nil, ack: false, &block)
         @connection_manager.connect(host: host || resolved_host, port: port, hostname: @hostname, ack: ack, &block)
       end

data/lib/fluent/plugin/out_forward/load_balancer.rb

@@ -56,7 +56,7 @@ module Fluent::Plugin
       end
 
       def rebuild_weight_array(nodes)
-        standby_nodes, regular_nodes = nodes.partition {|n|
+        standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|
           n.standby?
         }
 

data/lib/fluent/plugin/out_http.rb

@@ -37,6 +37,8 @@ module Fluent::Plugin
     config_param :proxy, :string, default: ENV['HTTP_PROXY'] || ENV['http_proxy']
     desc 'Content-Type for HTTP request'
     config_param :content_type, :string, default: nil
+    desc 'JSON array data format for HTTP request body'
+    config_param :json_array, :bool, default: false
     desc 'Additional headers for HTTP request'
     config_param :headers, :hash, default: nil
 
@@ -100,6 +102,13 @@ module Fluent::Plugin
       @proxy_uri = URI.parse(@proxy) if @proxy
       @formatter = formatter_create
       @content_type = setup_content_type unless @content_type
+
+      if @json_array
+        if @formatter_configs.first[:@type] != "json"
+          raise Fluent::ConfigError, "json_array option could be used with json formatter only"
+        end
+        define_singleton_method(:format, method(:format_json_array))
+      end
     end
 
     def multi_workers_ready?
@@ -114,6 +123,10 @@ module Fluent::Plugin
       @formatter.format(tag, time, record)
     end
 
+    def format_json_array(tag, time, record)
+      @formatter.format(tag, time, record) << ","
+    end
+
     def write(chunk)
       uri = parse_endpoint(chunk)
       req = create_request(chunk, uri)
@@ -128,7 +141,7 @@ module Fluent::Plugin
     def setup_content_type
       case @formatter_configs.first[:@type]
       when 'json'
-        'application/x-ndjson'
+        @json_array ? 'application/json' : 'application/x-ndjson'
       when 'csv'
         'text/csv'
       when 'tsv', 'ltsv'
@@ -202,7 +215,7 @@ module Fluent::Plugin
         req.basic_auth(@auth.username, @auth.password)
       end
       set_headers(req)
-      req.body = chunk.read
+      req.body = @json_array ? "[#{chunk.read.chop!}]" : chunk.read
       req
     end