fluentd 0.14.3 → 0.14.4

data/lib/fluent/plugin/in_tail.rb

@@ -67,14 +67,10 @@ module Fluent
     config_param :multiline_flush_interval, :time, default: nil
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
+    desc 'The encoding after conversion of the input.'
+    config_param :encoding, :string, default: nil
     desc 'The encoding of the input.'
-    config_param :encoding, default: nil do |encoding_name|
-      begin
-        Encoding.find(encoding_name)
-      rescue ArgumentError => e
-        raise ConfigError, e.message
-      end
-    end
+    config_param :from_encoding, :string, default: nil
     desc 'Add the log path being tailed to records. Specify the field name to be used.'
     config_param :path_key, :string, default: nil
 
@@ -95,6 +91,7 @@ module Fluent
 
       configure_parser(conf)
       configure_tag
+      configure_encoding
 
       @multiline_mode = conf['format'] =~ /multiline/
       @receive_handler = if @multiline_mode
@@ -120,6 +117,25 @@ module Fluent
       end
     end
 
+    def configure_encoding
+      unless @encoding
+        if @from_encoding
+          raise ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
+        end
+      end
+
+      @encoding = parse_encoding_param(@encoding) if @encoding
+      @from_encoding = parse_encoding_param(@from_encoding) if @from_encoding
+    end
+
+    def parse_encoding_param(encoding_name)
+      begin
+        Encoding.find(encoding_name) if encoding_name
+      rescue ArgumentError => e
+        raise ConfigError, e.message
+      end
+    end
+
     def start
       super
 
@@ -254,7 +270,13 @@ module Fluent
     def flush_buffer(tw)
       if lb = tw.line_buffer
         lb.chomp!
-        lb.force_encoding(@encoding) if @encoding
+        if @encoding
+          if @from_encoding
+            lb.encode!(@encoding, @from_encoding)
+          else
+            lb.force_encoding(@encoding)
+          end
+        end
         @parser.parse(lb) { |time, record|
           if time && record
             tag = if @tag_prefix || @tag_suffix
@@ -303,7 +325,13 @@ module Fluent
     def convert_line_to_event(line, es, tail_watcher)
       begin
         line.chomp!  # remove \n
-        line.force_encoding(@encoding) if @encoding
+        if @encoding
+          if @from_encoding
+            line.encode!(@encoding, @from_encoding)
+          else
+            line.force_encoding(@encoding)
+          end
+        end
         @parser.parse(line) { |time, record|
           if time && record
             record[@path_key] ||= tail_watcher.path unless @path_key.nil?

data/lib/fluent/plugin/out_forward.rb

@@ -89,23 +89,12 @@ module Fluent
 
     attr_reader :nodes
 
-    # backward compatibility
-    config_param :port, :integer, default: LISTEN_PORT
-    config_param :host, :string, default: nil
+    config_param :port, :integer, default: LISTEN_PORT, obsoleted: "User <server> section instead."
+    config_param :host, :string, default: nil, obsoleted: "Use <server> section instead."
 
     def configure(conf)
       super
 
-      # backward compatibility
-      if host = conf['host']
-        log.warn "'host' option in forward output is obsoleted. Use '<server> host xxx </server>' instead."
-        port = conf['port']
-        port = port ? port.to_i : LISTEN_PORT
-        element = conf.add_element('server')
-        element['host'] = host
-        element['port'] = port.to_s
-      end
-
       recover_sample_size = @recover_wait / @heartbeat_interval
 
       if @dns_round_robin

data/lib/fluent/plugin/output.rb

@@ -301,7 +301,7 @@ module Fluent
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
           @secondary.router = router if @secondary.has_router?
-          if self.class != @secondary.class
+          if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
             log.warn "secondary type should be same with primary one", primary: self.class.to_s, secondary: @secondary.class.to_s
           end
         else
@@ -384,6 +384,11 @@ module Fluent
         @secondary.start if @secondary
       end
 
+      def after_start
+        super
+        @secondary.after_start if @secondary
+      end
+
       def stop
         @secondary.stop if @secondary
         @buffer.stop if @buffering && @buffer
@@ -922,6 +927,11 @@ module Fluent
           interval = @buffer_config.flush_thread_interval
         end
 
+        while !self.after_started? && !self.stopped?
+          sleep 0.5
+        end
+        log.debug "enqueue_thread actually running"
+
         begin
           while @output_flush_threads_running
             now_int = Time.now.to_i
@@ -969,6 +979,11 @@ module Fluent
         clock_id = Process::CLOCK_MONOTONIC rescue Process::CLOCK_MONOTONIC_RAW
         state.next_time = Process.clock_gettime(clock_id) + flush_thread_interval
 
+        while !self.after_started? && !self.stopped?
+          sleep 0.5
+        end
+        log.debug "flush_thread actually running"
+
         begin
           # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase
           while @output_flush_threads_running

data/lib/fluent/plugin/storage_local.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 require 'fluent/plugin'
 require 'fluent/plugin/storage'
 

data/lib/fluent/plugin_helper/timer.rb

@@ -66,6 +66,7 @@ module Fluent
         def initialize(title, interval, repeat, log, checker, &callback)
           @title = title
           @callback = callback
+          @repeat = repeat
           @log = log
           @checker = checker
           super(interval, repeat)
@@ -76,8 +77,12 @@ module Fluent
         rescue => e
           @log.error "Unexpected error raised. Stopping the timer.", title: @title, error: e
           @log.error_backtrace
-          self.detach
+          detach
           @log.error "Timer detached.", title: @title
+        ensure
+          if attached?
+            detach unless @repeat
+          end
         end
       end
     end

data/lib/fluent/root_agent.rb

@@ -137,6 +137,9 @@ module Fluent
       lifecycle(desc: true) do |i| # instance
         i.start unless i.started?
       end
+      lifecycle(desc: true) do |i|
+        i.after_start unless i.after_started?
+      end
     end
 
     def flush!

data/lib/fluent/supervisor.rb

@@ -29,8 +29,10 @@ require 'shellwords'
 
 if Fluent.windows?
   require 'windows/library'
+  require 'windows/synchronize'
   require 'windows/system_info'
   include Windows::Library
+  include Windows::Synchronize
   include Windows::SystemInfo
   require 'win32/ipc'
   require 'win32/event'
@@ -48,16 +50,17 @@ module Fluent
       end
       install_supervisor_signal_handlers
 
+      if config[:signame]
+        @signame = config[:signame]
+        install_windows_event_handler
+      end
+
       socket_manager_path = ServerEngine::SocketManager::Server.generate_path
       ServerEngine::SocketManager::Server.open(socket_manager_path)
       ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
     end
 
     def after_run
-      if Time.now - @start_time < 1
-        $log.warn "process died within 1 second. exit."
-      end
-
       stop_rpc_server if @rpc_endpoint
     end
 
@@ -135,6 +138,22 @@ module Fluent
       end unless Fluent.windows?
     end
 
+    def install_windows_event_handler
+      Thread.new do
+        ev = Win32::Event.new(@signame)
+        begin
+          ev.reset
+          until WaitForSingleObject(ev.handle, 0) == WAIT_OBJECT_0
+            sleep 1
+          end
+          kill_worker
+          stop(true)
+        ensure
+          ev.close
+        end
+      end
+    end
+
     def supervisor_sighup_handler
       kill_worker
     end
@@ -220,10 +239,13 @@ module Fluent
       logger_initializer.init
       logger = $log
 
+      command_sender = Fluent.windows? ? "pipe" : "signal"
+
       # ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
       pid_path = params['daemonize']
       daemonize = !!params['daemonize']
       main_cmd = params['main_cmd']
+      signame = params['signame']
 
       se_config = {
           worker_type: 'spawn',
@@ -233,6 +255,8 @@ module Fluent
           log_stderr: false,
           enable_heartbeat: true,
           auto_heartbeat: false,
+          unrecoverable_exit_codes: [2],
+          stop_immediately_at_unrecoverable_exit: true,
           logger: logger,
           log: logger.out,
           log_path: log_path,
@@ -240,6 +264,7 @@ module Fluent
           logger_initializer: logger_initializer,
           chuser: chuser,
           chgroup: chgroup,
+          chumask: 0,
           suppress_repeated_stacktrace: suppress_repeated_stacktrace,
           daemonize: daemonize,
           rpc_endpoint: rpc_endpoint,
@@ -250,8 +275,10 @@ module Fluent
                                    WorkerModule.name,
                                    path,
                                    JSON.dump(params)],
+          command_sender: command_sender,
           fluentd_conf: fluentd_conf,
           main_cmd: main_cmd,
+          signame: signame,
       }
       if daemonize
         se_config[:pid_path] = pid_path
@@ -280,8 +307,8 @@ module Fluent
         if @path && @path != "-"
           @io = File.open(@path, "a")
           if @chuser || @chgroup
-            chuid = @chuser ? ServerEngine::Daemon.get_etc_passwd(@chuser).uid : nil
-            chgid = @chgroup ? ServerEngine::Daemon.get_etc_group(@chgroup).gid : nil
+            chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
+            chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
             File.chown(chuid, chgid, @path)
           end
         else
@@ -489,6 +516,7 @@ module Fluent
       params['chgroup'] = @chgroup
       params['use_v1_config'] = @use_v1_config
       params['suppress_repeated_stacktrace'] = @suppress_repeated_stacktrace
+      params['signame'] = @signame
 
       se = ServerEngine.create(ServerModule, WorkerModule){
         Fluent::Supervisor.load_config(@config_path, params)
@@ -527,6 +555,29 @@ module Fluent
       trap :USR1 do
         flush_buffer
       end unless Fluent.windows?
+
+      if Fluent.windows?
+        command_pipe = STDIN.dup
+        STDIN.reopen(File::NULL, "rb")
+        command_pipe.binmode
+        command_pipe.sync = true
+
+        Thread.new do
+          loop do
+            cmd = command_pipe.gets.chomp
+            case cmd
+            when "GRACEFUL_STOP", "IMMEDIATE_STOP"
+              $log.debug "fluentd main process get #{cmd} command"
+              @finished = true
+              $log.debug "getting start to shutdown main process"
+              Fluent::Engine.stop
+              break
+            else
+              $log.warn "fluentd main process get unknown command [#{cmd}]"
+            end
+          end
+        end
+      end
     end
 
     def flush_buffer
@@ -549,6 +600,8 @@ module Fluent
     def main_process(&block)
       Process.setproctitle("worker:#{@process_name}") if @process_name
 
+      configuration_error = false
+
       begin
         block.call
       rescue Fluent::ConfigError
@@ -562,7 +615,7 @@ module Fluent
           console.error "config error", file: @config_path, error: $!.to_s
           console.debug_backtrace
         end
-
+        configuration_error = true
       rescue
         $log.error "unexpected error", error: $!.to_s
         $log.error_backtrace
@@ -576,7 +629,7 @@ module Fluent
         end
       end
 
-      exit! 1
+      exit!(configuration_error ? 2 : 1)
     end
 
     def read_config
@@ -598,7 +651,7 @@ module Fluent
     end
 
     def change_privilege
-      ServerEngine::Daemon.change_privilege(@chuser, @chgroup)
+      ServerEngine::Privilege.change(@chuser, @chgroup)
     end
 
     def init_engine

data/lib/fluent/test/base.rb

@@ -30,7 +30,9 @@ module Fluent
           if block
             # Create new class for test w/ overwritten methods
             #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass.name
             klass = Class.new(klass)
+            klass.define_singleton_method("name") { klass_name }
             klass.module_eval(&block)
           end
           @instance = klass.new
@@ -62,6 +64,7 @@ module Fluent
       # num_waits is for checking thread status. This will be removed after improved plugin API
       def run(num_waits = 10, &block)
         @instance.start
+        @instance.after_start
         begin
           # wait until thread starts
           num_waits.times { sleep 0.05 }

data/lib/fluent/test/driver/base.rb

@@ -31,7 +31,9 @@ module Fluent
             if block
               # Create new class for test w/ overwritten methods
               #   klass.dup is worse because its ancestors does NOT include original class name
+              klass_name = klass.name
               klass = Class.new(klass)
+              klass.define_singleton_method("name") { klass_name }
               klass.module_eval(&block)
             end
             @instance = klass.new
@@ -86,6 +88,9 @@ module Fluent
             @instance.start
             instance_hook_after_started
           end
+          unless @instance.after_started?
+            @instance.after_start
+          end
         end
 
         def instance_hook_after_started

data/lib/fluent/test/formatter_test.rb

@@ -26,7 +26,9 @@ module Fluent
           if block
             # Create new class for test w/ overwritten methods
             #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass_or_str.name
             klass_or_str = Class.new(klass_or_str)
+            klass_or_str.define_singleton_method("name") { klass_name }
             klass_or_str.module_eval(&block)
           end
           @instance = klass_or_str.new

data/lib/fluent/test/parser_test.rb

@@ -25,7 +25,9 @@ module Fluent
           if block
             # Create new class for test w/ overwritten methods
             #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass_or_str.name
             klass_or_str = Class.new(klass_or_str)
+            klass_or_str.define_singleton_method("name") { klass_name }
             klass_or_str.module_eval(&block)
           end
           case klass_or_str.instance_method(:initialize).arity

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.3'
+  VERSION = '0.14.4'
 
 end