fluentd 1.10.2-x64-mingw32 → 1.11.2-x64-mingw32

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -223,7 +223,7 @@ module Fluent::Plugin
         opts = {with_config: false, with_retry: false}
         timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
           es = Fluent::MultiEventStream.new
-          now = Fluent::Engine.now
+          now = Fluent::EventTime.now
           plugins_info_all(opts).each { |record|
             es.add(now, record)
           }

data/lib/fluent/plugin/in_sample.rb

@@ -0,0 +1,141 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'json'
+
+require 'fluent/plugin/input'
+require 'fluent/config/error'
+
+module Fluent::Plugin
+  class SampleInput < Input
+    Fluent::Plugin.register_input('sample', self)
+    Fluent::Plugin.register_input('dummy', self)
+
+    helpers :thread, :storage
+
+    BIN_NUM = 10
+    DEFAULT_STORAGE_TYPE = 'local'
+
+    desc "The value is the tag assigned to the generated events."
+    config_param :tag, :string
+    desc "The number of events in event stream of each emits."
+    config_param :size, :integer, default: 1
+    desc "It configures how many events to generate per second."
+    config_param :rate, :integer, default: 1
+    desc "If specified, each generated event has an auto-incremented key field."
+    config_param :auto_increment_key, :string, default: nil
+    desc "The boolean to suspend-and-resume incremental value after restart"
+    config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
+    desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
+    config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
+      begin
+        parsed = JSON.parse(val)
+      rescue JSON::ParserError => ex
+        # Fluent::ConfigParseError, "got incomplete JSON" will be raised
+        # at literal_parser.rb with --use-v1-config, but I had to
+        # take care at here for the case of --use-v0-config.
+        raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
+      end
+      sample = parsed.is_a?(Array) ? parsed : [parsed]
+      sample.each_with_index do |e, i|
+        raise Fluent::ConfigError, "#{i}th element of sample, #{e}, is not a hash" unless e.is_a?(Hash)
+      end
+      sample
+    end
+
+    def initialize
+      super
+      @storage = nil
+    end
+
+    def configure(conf)
+      super
+      @sample_index = 0
+      config = conf.elements.select{|e| e.name == 'storage' }.first
+      @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
+      # keep 'dummy' to avoid breaking changes for existing environment. Change it in fluentd v2
+      @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
+
+      if @auto_increment_key && !@storage.get(:auto_increment_value)
+        @storage.put(:auto_increment_value, -1)
+      end
+
+      thread_create(:sample_input, &method(:run))
+    end
+
+    def run
+      batch_num    = (@rate / BIN_NUM).to_i
+      residual_num = (@rate % BIN_NUM)
+      while thread_current_running?
+        current_time = Time.now.to_i
+        BIN_NUM.times do
+          break unless (thread_current_running? && Time.now.to_i <= current_time)
+          wait(0.1) { emit(batch_num) }
+        end
+        emit(residual_num) if thread_current_running?
+        # wait for next second
+        while thread_current_running? && Time.now.to_i <= current_time
+          sleep 0.01
+        end
+      end
+    end
+
+    def emit(num)
+      begin
+        if @size > 1
+          num.times do
+            router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
+          end
+        else
+          num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
+        end
+      rescue => _
+        # ignore all errors not to stop emits by emit errors
+      end
+    end
+
+    def generate
+      d = @sample[@sample_index]
+      unless d
+        @sample_index = 0
+        d = @sample[@sample_index]
+      end
+      @sample_index += 1
+      if @auto_increment_key
+        d = d.dup
+        d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
+      end
+      d
+    end
+
+    def wait(time)
+      start_time = Time.now
+      yield
+      sleep_time = time - (Time.now - start_time)
+      sleep sleep_time if sleep_time > 0
+    end
+  end
+end

data/lib/fluent/plugin/in_tail.rb

@@ -136,7 +136,7 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
       end
 
-      @paths = @path.split(@path_delimiter).map(&:strip)
+      @paths = @path.split(@path_delimiter).map(&:strip).uniq
       if @paths.empty?
         raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
       end
@@ -271,9 +271,9 @@ module Fluent::Plugin
                 end
               else
                 if is_file
-                  unless @ignore_list.include?(path)
+                  unless @ignore_list.include?(p)
                     log.warn "#{p} unreadable. It is excluded and would be examined next time."
-                    @ignore_list << path if @ignore_repeated_permission_error
+                    @ignore_list << p if @ignore_repeated_permission_error
                   end
                 end
                 false
@@ -296,7 +296,7 @@ module Fluent::Plugin
                end
         path.include?('*') ? Dir.glob(path) : path
       }.flatten.uniq
-      paths - excluded
+      paths.uniq - excluded
     end
 
     # in_tail with '*' path doesn't check rotation file equality at refresh phase.

data/lib/fluent/plugin/in_unix.rb

@@ -14,50 +14,68 @@
 #    limitations under the License.
 #
 
-require 'fileutils'
-require 'socket'
+require 'fluent/env'
+require 'fluent/plugin/input'
+require 'fluent/msgpack_factory'
 
 require 'cool.io'
 require 'yajl'
+require 'fileutils'
+require 'socket'
+
+module Fluent::Plugin
+  # TODO: This plugin will be 3rd party plugin
+  class UnixInput < Input
+    Fluent::Plugin.register_input('unix', self)
+
+    helpers :event_loop
+
+    def initialize
+      super
 
-require 'fluent/input'
-require 'fluent/event'
+      @lsock = nil
+    end
 
-module Fluent
-  # obsolete
-  class StreamInput < Input
-    config_param :blocking_timeout, :time, default: 0.5
+    desc 'The path to your Unix Domain Socket.'
+    config_param :path, :string, default: Fluent::DEFAULT_SOCKET_PATH
+    desc 'The backlog of Unix Domain Socket.'
+    config_param :backlog, :integer, default: nil
+    desc "New tag instead of incoming tag"
+    config_param :tag, :string, default: nil
+
+    def configure(conf)
+      super
+    end
 
     def start
       super
 
-      @loop = Coolio::Loop.new
       @lsock = listen
-      @loop.attach(@lsock)
-      @thread = Thread.new(&method(:run))
+      event_loop_attach(@lsock)
     end
 
     def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @lsock.close
-      @thread.join
+      if @lsock
+        event_loop_detach(@lsock)
+        @lsock.close
+      end
 
       super
     end
 
-    #def listen
-    #end
+    def listen
+      if File.exist?(@path)
+        log.warn "Found existing '#{@path}'. Remove this file for in_unix plugin"
+        File.unlink(@path)
+      end
+      FileUtils.mkdir_p(File.dirname(@path))
 
-    def run
-      @loop.run(@blocking_timeout)
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
+      log.info "listening fluent socket on #{@path}"
+      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
+      s.listen(@backlog) unless @backlog.nil?
+      s
     end
 
-    private
-
     # message Entry {
     #   1: long time
     #   2: object record
@@ -79,23 +97,27 @@ module Fluent
     #   3: object record
     # }
     def on_message(msg)
-      # TODO format error
-      tag = msg[0].to_s
+      unless msg.is_a?(Array)
+        log.warn "incoming data is broken:", msg: msg
+        return
+      end
+
+      tag = @tag || (msg[0].to_s)
       entries = msg[1]
 
-      if entries.class == String
+      case entries
+      when String
         # PackedForward
-        es = MessagePackEventStream.new(entries)
+        es = Fluent::MessagePackEventStream.new(entries)
         router.emit_stream(tag, es)
 
-      elsif entries.class == Array
+      when Array
         # Forward
-        es = MultiEventStream.new
+        es = Fluent::MultiEventStream.new
         entries.each {|e|
           record = e[1]
           next if record.nil?
-          time = e[0]
-          time = (now ||= Engine.now) if time.to_i == 0
+          time = convert_time(e[0])
           es.add(time, record)
         }
         router.emit_stream(tag, es)
@@ -105,25 +127,28 @@ module Fluent
         record = msg[2]
         return if record.nil?
 
-        time = msg[1]
-        time = Engine.now if time.to_i == 0
+        time = convert_time(msg[1])
         router.emit(tag, time, record)
       end
     end
 
+    def convert_time(time)
+      case time
+      when nil, 0
+        Fluent::EventTime.now
+      when Fluent::EventTime
+        time
+      else
+        Fluent::EventTime.from_time(Time.at(time))
+      end
+    end
+
     class Handler < Coolio::Socket
       def initialize(io, log, on_message)
         super(io)
-        if io.is_a?(TCPSocket)
-          opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-        end
+
         @on_message = on_message
         @log = log
-        @log.trace {
-          remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername) rescue nil
-          "accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
-        }
       end
 
       def on_connect
@@ -131,13 +156,13 @@ module Fluent
 
       def on_read(data)
         first = data[0]
-        if first == '{' || first == '['
+        if first == '{'.freeze || first == '['.freeze
           m = method(:on_read_json)
-          @y = Yajl::Parser.new
-          @y.on_parse_complete = @on_message
+          @parser = Yajl::Parser.new
+          @parser.on_parse_complete = @on_message
         else
           m = method(:on_read_msgpack)
-          @u = Fluent::MessagePackFactory.msgpack_unpacker
+          @parser = Fluent::MessagePackFactory.msgpack_unpacker
         end
 
         singleton_class.module_eval do
@@ -147,17 +172,17 @@ module Fluent
       end
 
       def on_read_json(data)
-        @y << data
-      rescue
-        @log.error "unexpected error", error: $!.to_s
+        @parser << data
+      rescue => e
+        @log.error "unexpected error in json payload", error: e.to_s
         @log.error_backtrace
         close
       end
 
       def on_read_msgpack(data)
-        @u.feed_each(data, &@on_message)
-      rescue
-        @log.error "unexpected error", error: $!.to_s
+        @parser.feed_each(data, &@on_message)
+      rescue => e
+        @log.error "unexpected error in msgpack payload", error: e.to_s
         @log.error_backtrace
         close
       end
@@ -167,29 +192,4 @@ module Fluent
       end
     end
   end
-
-  class UnixInput < StreamInput
-    Plugin.register_input('unix', self)
-
-    desc 'The path to your Unix Domain Socket.'
-    config_param :path, :string, default: DEFAULT_SOCKET_PATH
-    desc 'The backlog of Unix Domain Socket.'
-    config_param :backlog, :integer, default: nil
-
-    def configure(conf)
-      super
-      #log.warn "'unix' input is obsoleted and will be removed. Use 'forward' instead."
-    end
-
-    def listen
-      if File.exist?(@path)
-        File.unlink(@path)
-      end
-      FileUtils.mkdir_p File.dirname(@path)
-      log.info "listening fluent socket on #{@path}"
-      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
-      s.listen(@backlog) unless @backlog.nil?
-      s
-    end
-  end
 end

data/lib/fluent/plugin/out_file.rb

@@ -155,7 +155,7 @@ module Fluent::Plugin
         dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
         dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
 
-        test_chunk1 = chunk_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
+        test_chunk1 = chunk_for_test(dummy_tag, Fluent::EventTime.now, dummy_record)
         test_path = extract_placeholders(@path_template, test_chunk1)
         unless ::Fluent::FileUtil.writable_p?(test_path)
           raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"

data/lib/fluent/plugin/out_forward.rb

@@ -382,7 +382,12 @@ module Fluent::Plugin
           cert_logical_store_name: @tls_cert_logical_store_name,
           cert_use_enterprise_store: @tls_cert_use_enterprise_store,
 
-          linger_timeout: @send_timeout,
+          # Enabling SO_LINGER causes tcp port exhaustion on Windows.
+          # This is because dynamic ports are only 16384 (from 49152 to 65535) and
+          # expiring SO_LINGER enabled ports should wait 4 minutes
+          # where set by TcpTimeDelay. Its default value is 4 minutes.
+          # So, we should disable SO_LINGER on Windows to prevent flood of waiting ports.
+          linger_timeout: Fluent.windows? ? nil : @send_timeout,
           send_timeout: @send_timeout,
           recv_timeout: @ack_response_timeout,
           connect_timeout: @connect_timeout,
@@ -704,7 +709,7 @@ module Fluent::Plugin
           @resolved_host ||= resolve_dns!
 
         else
-          now = Fluent::Engine.now
+          now = Fluent::EventTime.now
           rh = @resolved_host
           if !rh || now - @resolved_time >= @sender.expire_dns_cache
             rh = @resolved_host = resolve_dns!