fluentd 0.14.2 → 0.14.5

data/lib/fluent/plugin/out_null.rb

@@ -18,10 +18,47 @@ require 'fluent/plugin/output'
 
 module Fluent::Plugin
   class NullOutput < Output
+    # This plugin is for tests of non-buffered/buffered plugins
     Fluent::Plugin.register_output('null', self)
 
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
+    def prefer_buffered_processing
+      false
+    end
+
+    def prefer_delayed_commit
+      @delayed
+    end
+
+    attr_accessor :feed_proc, :delayed
+
+    def initialize
+      super
+      @delayed = false
+      @feed_proc = nil
+    end
+
     def process(tag, es)
       # Do nothing
     end
+
+    def write(chunk)
+      if @feed_proc
+        @feed_proc.call(chunk)
+      end
+    end
+
+    def try_write(chunk)
+      if @feed_proc
+        @feed_proc.call(chunk)
+      end
+      # not to commit chunks for testing
+      # commit_write(chunk.unique_id)
+    end
   end
 end

data/lib/fluent/plugin/out_secondary_file.rb

@@ -0,0 +1,128 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "fileutils"
+require "fluent/plugin/file_util"
+require "fluent/plugin/output"
+require "fluent/config/error"
+
+module Fluent::Plugin
+  class SecondaryFileOutput < Output
+    Fluent::Plugin.register_output("secondary_file", self)
+
+    FILE_PERMISSION = 0644
+    DIR_PERMISSION = 0755
+    PLACEHOLDER_REGEX = /\${(tag(\[\d+\])?|[\w.@-]+)}/
+
+    desc "The directory path of the output file."
+    config_param :directory, :string
+    desc "The basename of the output file."
+    config_param :basename, :string, default: "dump.bin"
+    desc "The flushed chunk is appended to existence file or not."
+    config_param :append, :bool, default: false
+    config_param :compress, :enum, list: [:text, :gzip], default: :text
+
+    def configure(conf)
+      super
+
+      unless @as_secondary
+        raise Fluent::ConfigError, "This plugin can only be used in the <secondary> section"
+      end
+
+      if @basename.include?("/")
+        raise Fluent::ConfigError, "basename should not include `/`"
+      end
+
+      @path_without_suffix = File.join(@directory, @basename)
+      validate_compatible_with_primary_buffer!(@path_without_suffix)
+
+      @suffix = case @compress
+                when :text
+                  ""
+                when :gzip
+                  ".gz"
+                end
+
+      test_path = @path_without_suffix
+      unless Fluent::FileUtil.writable_p?(test_path)
+        raise Fluent::ConfigError, "out_secondary_file: `#{@directory}` should be writable"
+      end
+
+      @dir_perm = system_config.dir_permission || DIR_PERMISSION
+      @file_perm = system_config.file_permission || FILE_PERMISSION
+    end
+
+    def write(chunk)
+      path_without_suffix = extract_placeholders(@path_without_suffix, chunk.metadata)
+      path = generate_path(path_without_suffix)
+      FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
+
+      case @compress
+      when :text
+        File.open(path, "ab", @file_perm) {|f|
+          f.flock(File::LOCK_EX)
+          chunk.write_to(f)
+        }
+      when :gzip
+        File.open(path, "ab", @file_perm) {|f|
+          f.flock(File::LOCK_EX)
+          gz = Zlib::GzipWriter.new(f)
+          chunk.write_to(gz)
+          gz.close
+        }
+      end
+
+      path
+    end
+
+    private
+
+    def validate_compatible_with_primary_buffer!(path_without_suffix)
+      placeholders = path_without_suffix.scan(PLACEHOLDER_REGEX).flat_map(&:first) # to trim suffix [\d+]
+
+      if !@chunk_key_time && has_time_format?(path_without_suffix)
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder, remove time formats, like `%Y%m%d`, from basename or directory"
+      end
+
+      if !@chunk_key_tag && (ph = placeholders.find { |placeholder| placeholder.match(/tag(\[\d+\])?/) })
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove tag placeholder, like `${tag}`, from basename or directory"
+      end
+
+      vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) }
+
+      if ph = vars.find { |v| !@chunk_keys.include?(v) }
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove variable placeholder, like `${varname}`, from basename or directory"
+      end
+    end
+
+    def has_time_format?(str)
+      str != Time.now.strftime(str)
+    end
+
+    def generate_path(path_without_suffix)
+      if @append
+        "#{path_without_suffix}#{@suffix}"
+      else
+        i = 0
+        loop do
+          path = "#{path_without_suffix}.#{i}#{@suffix}"
+          return path unless File.exist?(path)
+          i += 1
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_stdout.rb

@@ -24,25 +24,61 @@ module Fluent::Plugin
 
     DEFAULT_FORMAT_TYPE = 'json'
 
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
     config_section :format do
       config_set_default :@type, DEFAULT_FORMAT_TYPE
     end
 
+    def prefer_buffered_processing
+      false
+    end
+
+    def prefer_delayed_commit
+      @delayed
+    end
+
+    attr_accessor :delayed
+
+    def initialize
+      super
+      @delayed = false
+    end
+
     def configure(conf)
       if conf['output_type'] && !conf['format']
         conf['format'] = conf['output_type']
       end
       compat_parameters_convert(conf, :inject, :formatter)
+
       super
+
       @formatter = formatter_create(conf: conf.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
     end
 
     def process(tag, es)
       es.each {|time,record|
-        r = inject_values_to_record(tag, time, record)
-        $log.write "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, r).chomp}\n"
+        $log.write(format(tag, time, record))
       }
       $log.flush
     end
+
+    def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
+      "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
+    end
+
+    def write(chunk)
+      chunk.write_to($log)
+    end
+
+    def try_write(chunk)
+      chunk.write_to($log)
+      commit_write(chunk.unique_id)
+    end
   end
 end

data/lib/fluent/plugin/output.rb

@@ -138,7 +138,7 @@ module Fluent
         end
       end
 
-      attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout
+      attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
       attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
 
       # for tests
@@ -185,13 +185,21 @@ module Fluent
         @simple_chunking = nil
         @chunk_keys = @chunk_key_time = @chunk_key_tag = nil
         @flush_mode = nil
+        @timekey_zone = nil
       end
 
       def acts_as_secondary(primary)
         @as_secondary = true
         @primary_instance = primary
+        @chunk_keys = @primary_instance.chunk_keys || []
+        @chunk_key_tag = @primary_instance.chunk_key_tag || false
+        if @primary_instance.chunk_key_time
+          @chunk_key_time = @primary_instance.chunk_key_time
+          @timekey_zone = @primary_instance.timekey_zone
+          @output_time_formatter_cache = {}
+        end
+
         (class << self; self; end).module_eval do
-          define_method(:extract_placeholders){ |str, metadata| @primary_instance.extract_placeholders(str, metadata) }
           define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) }
           define_method(:rollback_write){ |chunk_id| @primary_instance.rollback_write(chunk_id) }
         end
@@ -251,7 +259,7 @@ module Fluent
           if @chunk_key_time
             raise Fluent::ConfigError, "<buffer ...> argument includes 'time', but timekey is not configured" unless @buffer_config.timekey
             Fluent::Timezone.validate!(@buffer_config.timekey_zone)
-            @buffer_config.timekey_zone = '+0000' if @buffer_config.timekey_use_utc
+            @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
             @output_time_formatter_cache = {}
           end
 
@@ -301,7 +309,7 @@ module Fluent
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
           @secondary.router = router if @secondary.has_router?
-          if self.class != @secondary.class
+          if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
             log.warn "secondary type should be same with primary one", primary: self.class.to_s, secondary: @secondary.class.to_s
           end
         else
@@ -384,6 +392,11 @@ module Fluent
         @secondary.start if @secondary
       end
 
+      def after_start
+        super
+        @secondary.after_start if @secondary
+      end
+
       def stop
         @secondary.stop if @secondary
         @buffer.stop if @buffering && @buffer
@@ -472,13 +485,13 @@ module Fluent
 
       # TODO: optimize this code
       def extract_placeholders(str, metadata)
-        if metadata.timekey.nil? && metadata.tag.nil? && metadata.variables.nil?
+        if metadata.empty?
           str
         else
           rvalue = str
           # strftime formatting
           if @chunk_key_time # this section MUST be earlier than rest to use raw 'str'
-            @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@buffer_config.timekey_zone, str)
+            @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)
             rvalue = @output_time_formatter_cache[str].call(metadata.timekey)
           end
           # ${tag}, ${tag[0]}, ${tag[1]}, ...
@@ -922,6 +935,11 @@ module Fluent
           interval = @buffer_config.flush_thread_interval
         end
 
+        while !self.after_started? && !self.stopped?
+          sleep 0.5
+        end
+        log.debug "enqueue_thread actually running"
+
         begin
           while @output_flush_threads_running
             now_int = Time.now.to_i
@@ -969,6 +987,11 @@ module Fluent
         clock_id = Process::CLOCK_MONOTONIC rescue Process::CLOCK_MONOTONIC_RAW
         state.next_time = Process.clock_gettime(clock_id) + flush_thread_interval
 
+        while !self.after_started? && !self.stopped?
+          sleep 0.5
+        end
+        log.debug "flush_thread actually running"
+
         begin
           # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase
           while @output_flush_threads_running

data/lib/fluent/plugin/storage_local.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 require 'fluent/plugin'
 require 'fluent/plugin/storage'
 

data/lib/fluent/plugin_helper/timer.rb

@@ -66,6 +66,7 @@ module Fluent
         def initialize(title, interval, repeat, log, checker, &callback)
           @title = title
           @callback = callback
+          @repeat = repeat
           @log = log
           @checker = checker
           super(interval, repeat)
@@ -76,8 +77,12 @@ module Fluent
         rescue => e
           @log.error "Unexpected error raised. Stopping the timer.", title: @title, error: e
           @log.error_backtrace
-          self.detach
+          detach
           @log.error "Timer detached.", title: @title
+        ensure
+          if attached?
+            detach unless @repeat
+          end
         end
       end
     end

data/lib/fluent/root_agent.rb

@@ -91,7 +91,7 @@ module Fluent
       else
         conf.elements(name: 'source').each { |e|
           type = e['@type']
-          raise ConfigError, "Missing 'type' parameter on <source> directive" unless type
+          raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
           add_source(type, e)
         }
       end
@@ -137,6 +137,9 @@ module Fluent
       lifecycle(desc: true) do |i| # instance
         i.start unless i.started?
       end
+      lifecycle(desc: true) do |i|
+        i.after_start unless i.after_started?
+      end
     end
 
     def flush!

data/lib/fluent/supervisor.rb

@@ -29,8 +29,10 @@ require 'shellwords'
 
 if Fluent.windows?
   require 'windows/library'
+  require 'windows/synchronize'
   require 'windows/system_info'
   include Windows::Library
+  include Windows::Synchronize
   include Windows::SystemInfo
   require 'win32/ipc'
   require 'win32/event'
@@ -48,16 +50,17 @@ module Fluent
       end
       install_supervisor_signal_handlers
 
+      if config[:signame]
+        @signame = config[:signame]
+        install_windows_event_handler
+      end
+
       socket_manager_path = ServerEngine::SocketManager::Server.generate_path
       ServerEngine::SocketManager::Server.open(socket_manager_path)
       ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
     end
 
     def after_run
-      if Time.now - @start_time < 1
-        $log.warn "process died within 1 second. exit."
-      end
-
       stop_rpc_server if @rpc_endpoint
     end
 
@@ -135,6 +138,22 @@ module Fluent
       end unless Fluent.windows?
     end
 
+    def install_windows_event_handler
+      Thread.new do
+        ev = Win32::Event.new(@signame)
+        begin
+          ev.reset
+          until WaitForSingleObject(ev.handle, 0) == WAIT_OBJECT_0
+            sleep 1
+          end
+          kill_worker
+          stop(true)
+        ensure
+          ev.close
+        end
+      end
+    end
+
     def supervisor_sighup_handler
       kill_worker
     end
@@ -220,10 +239,13 @@ module Fluent
       logger_initializer.init
       logger = $log
 
+      command_sender = Fluent.windows? ? "pipe" : "signal"
+
       # ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
       pid_path = params['daemonize']
       daemonize = !!params['daemonize']
       main_cmd = params['main_cmd']
+      signame = params['signame']
 
       se_config = {
           worker_type: 'spawn',
@@ -233,6 +255,8 @@ module Fluent
           log_stderr: false,
           enable_heartbeat: true,
           auto_heartbeat: false,
+          unrecoverable_exit_codes: [2],
+          stop_immediately_at_unrecoverable_exit: true,
           logger: logger,
           log: logger.out,
           log_path: log_path,
@@ -240,6 +264,7 @@ module Fluent
           logger_initializer: logger_initializer,
           chuser: chuser,
           chgroup: chgroup,
+          chumask: 0,
           suppress_repeated_stacktrace: suppress_repeated_stacktrace,
           daemonize: daemonize,
           rpc_endpoint: rpc_endpoint,
@@ -250,8 +275,10 @@ module Fluent
                                    WorkerModule.name,
                                    path,
                                    JSON.dump(params)],
+          command_sender: command_sender,
           fluentd_conf: fluentd_conf,
           main_cmd: main_cmd,
+          signame: signame,
       }
       if daemonize
         se_config[:pid_path] = pid_path
@@ -280,8 +307,8 @@ module Fluent
         if @path && @path != "-"
           @io = File.open(@path, "a")
           if @chuser || @chgroup
-            chuid = @chuser ? ServerEngine::Daemon.get_etc_passwd(@chuser).uid : nil
-            chgid = @chgroup ? ServerEngine::Daemon.get_etc_group(@chgroup).gid : nil
+            chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
+            chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
             File.chown(chuid, chgid, @path)
           end
         else
@@ -489,6 +516,7 @@ module Fluent
       params['chgroup'] = @chgroup
       params['use_v1_config'] = @use_v1_config
       params['suppress_repeated_stacktrace'] = @suppress_repeated_stacktrace
+      params['signame'] = @signame
 
       se = ServerEngine.create(ServerModule, WorkerModule){
         Fluent::Supervisor.load_config(@config_path, params)
@@ -527,6 +555,29 @@ module Fluent
       trap :USR1 do
         flush_buffer
       end unless Fluent.windows?
+
+      if Fluent.windows?
+        command_pipe = STDIN.dup
+        STDIN.reopen(File::NULL, "rb")
+        command_pipe.binmode
+        command_pipe.sync = true
+
+        Thread.new do
+          loop do
+            cmd = command_pipe.gets.chomp
+            case cmd
+            when "GRACEFUL_STOP", "IMMEDIATE_STOP"
+              $log.debug "fluentd main process get #{cmd} command"
+              @finished = true
+              $log.debug "getting start to shutdown main process"
+              Fluent::Engine.stop
+              break
+            else
+              $log.warn "fluentd main process get unknown command [#{cmd}]"
+            end
+          end
+        end
+      end
     end
 
     def flush_buffer
@@ -549,6 +600,8 @@ module Fluent
     def main_process(&block)
       Process.setproctitle("worker:#{@process_name}") if @process_name
 
+      configuration_error = false
+
       begin
         block.call
       rescue Fluent::ConfigError
@@ -562,7 +615,7 @@ module Fluent
           console.error "config error", file: @config_path, error: $!.to_s
           console.debug_backtrace
         end
-
+        configuration_error = true
       rescue
         $log.error "unexpected error", error: $!.to_s
         $log.error_backtrace
@@ -576,7 +629,7 @@ module Fluent
         end
       end
 
-      exit! 1
+      exit!(configuration_error ? 2 : 1)
     end
 
     def read_config
@@ -598,7 +651,7 @@ module Fluent
     end
 
     def change_privilege
-      ServerEngine::Daemon.change_privilege(@chuser, @chgroup)
+      ServerEngine::Privilege.change(@chuser, @chgroup)
     end
 
     def init_engine

data/lib/fluent/test/base.rb

@@ -30,7 +30,9 @@ module Fluent
           if block
             # Create new class for test w/ overwritten methods
             #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass.name
             klass = Class.new(klass)
+            klass.define_singleton_method("name") { klass_name }
             klass.module_eval(&block)
           end
           @instance = klass.new
@@ -62,6 +64,7 @@ module Fluent
       # num_waits is for checking thread status. This will be removed after improved plugin API
       def run(num_waits = 10, &block)
         @instance.start
+        @instance.after_start
         begin
           # wait until thread starts
           num_waits.times { sleep 0.05 }

data/lib/fluent/test/driver/base.rb

@@ -31,7 +31,9 @@ module Fluent
             if block
               # Create new class for test w/ overwritten methods
               #   klass.dup is worse because its ancestors does NOT include original class name
+              klass_name = klass.name
               klass = Class.new(klass)
+              klass.define_singleton_method("name") { klass_name }
               klass.module_eval(&block)
             end
             @instance = klass.new
@@ -86,6 +88,9 @@ module Fluent
             @instance.start
             instance_hook_after_started
           end
+          unless @instance.after_started?
+            @instance.after_start
+          end
         end
 
         def instance_hook_after_started

data/lib/fluent/test/formatter_test.rb

@@ -26,7 +26,9 @@ module Fluent
           if block
             # Create new class for test w/ overwritten methods
             #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass_or_str.name
             klass_or_str = Class.new(klass_or_str)
+            klass_or_str.define_singleton_method("name") { klass_name }
             klass_or_str.module_eval(&block)
           end
           @instance = klass_or_str.new

data/lib/fluent/test/parser_test.rb

@@ -25,7 +25,9 @@ module Fluent
           if block
             # Create new class for test w/ overwritten methods
             #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass_or_str.name
             klass_or_str = Class.new(klass_or_str)
+            klass_or_str.define_singleton_method("name") { klass_name }
             klass_or_str.module_eval(&block)
           end
           case klass_or_str.instance_method(:initialize).arity

data/lib/fluent/test/startup_shutdown.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'serverengine'
+
+module Fluent
+  module Test
+    module StartupShutdown
+      def startup
+        socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      end
+
+      def shutdown
+        @server.close
+      end
+    end
+  end
+end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.2'
+  VERSION = '0.14.5'
 
 end