fluentd 0.14.11 → 0.14.12

data/lib/fluent/plugin/out_null.rb

@@ -21,6 +21,9 @@ module Fluent::Plugin
     # This plugin is for tests of non-buffered/buffered plugins
     Fluent::Plugin.register_output('null', self)
 
+    desc "The parameter for testing to simulate output plugin which never succeed to flush."
+    config_param :never_flush, :bool, default: false
+
     config_section :buffer do
       config_set_default :chunk_keys, ['tag']
       config_set_default :flush_at_shutdown, true
@@ -43,17 +46,24 @@ module Fluent::Plugin
       @feed_proc = nil
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def process(tag, es)
+      raise "failed to flush" if @never_flush
       # Do nothing
     end
 
     def write(chunk)
+      raise "failed to flush" if @never_flush
       if @feed_proc
         @feed_proc.call(chunk)
       end
     end
 
     def try_write(chunk)
+      raise "failed to flush" if @never_flush
       if @feed_proc
         @feed_proc.call(chunk)
       end

data/lib/fluent/plugin/out_relabel.rb

@@ -21,6 +21,10 @@ module Fluent::Plugin
     Fluent::Plugin.register_output('relabel', self)
     helpers :event_emitter
 
+    def multi_workers_ready?
+      true
+    end
+
     def process(tag, es)
       router.emit_stream(tag, es)
     end

data/lib/fluent/plugin/out_roundrobin.rb

@@ -42,6 +42,10 @@ module Fluent::Plugin
       @rand_seed = Random.new.seed
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def start
       super
       rebuild_weight_array

data/lib/fluent/plugin/out_secondary_file.rb

@@ -65,6 +65,11 @@ module Fluent::Plugin
       @file_perm = system_config.file_permission || FILE_PERMISSION
     end
 
+    def multi_workers_ready?
+      ### TODO: add hack to synchronize for multi workers
+      true
+    end
+
     def write(chunk)
       path_without_suffix = extract_placeholders(@path_without_suffix, chunk.metadata)
       path = generate_path(path_without_suffix)

data/lib/fluent/plugin/out_stdout.rb

@@ -41,6 +41,10 @@ module Fluent::Plugin
       false
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     attr_accessor :formatter
 
     def configure(conf)
@@ -52,6 +56,7 @@ module Fluent::Plugin
     end
 
     def process(tag, es)
+      es = inject_values_to_event_stream(tag, es)
       es.each {|time,record|
         $log.write(format(tag, time, record))
       }

data/lib/fluent/plugin/output.rb

@@ -20,6 +20,7 @@ require 'fluent/plugin_id'
 require 'fluent/plugin_helper'
 require 'fluent/timezone'
 require 'fluent/unique_id'
+require 'fluent/clock'
 
 require 'time'
 require 'monitor'
@@ -40,8 +41,6 @@ module Fluent
 
       CHUNKING_FIELD_WARN_NUM = 4
 
-      PROCESS_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
-
       config_param :time_as_integer, :bool, default: false
       desc 'The threshold to show slow flush logs'
       config_param :slow_flush_log_threshold, :float, default: 20.0
@@ -141,6 +140,10 @@ module Fluent
         true
       end
 
+      def multi_workers_ready?
+        false
+      end
+
       # Internal states
       FlushThreadState = Struct.new(:thread, :next_clock)
       DequeuedChunkInfo = Struct.new(:chunk_id, :time, :timeout) do
@@ -782,13 +785,19 @@ module Fluent
         begin
           block.call
         rescue Fluent::Plugin::Buffer::BufferOverflowError
-          log.warn "failed to write data into buffer by buffer overflow"
+          log.warn "failed to write data into buffer by buffer overflow", action: @buffer_config.overflow_action
           case @buffer_config.overflow_action
           when :throw_exception
             raise
           when :block
             log.debug "buffer.write is now blocking"
             until @buffer.storable?
+              if self.stopped?
+                log.error "breaking block behavior to shutdown Fluentd"
+                # to break infinite loop to exit Fluentd process
+                raise
+              end
+              log.trace "sleeping until buffer can store more data"
               sleep 1
             end
             log.debug "retrying buffer.write after blocked operation"
@@ -973,7 +982,7 @@ module Fluent
         chunk = @buffer.dequeue_chunk
         return unless chunk
 
-        log.debug "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id)
+        log.trace "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id)
 
         output = self
         using_secondary = false
@@ -987,7 +996,7 @@ module Fluent
         end
 
         begin
-          chunk_write_start = Process.clock_gettime(PROCESS_CLOCK_ID)
+          chunk_write_start = Fluent::Clock.now
 
           if output.delayed_commit
             log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
@@ -1029,7 +1038,7 @@ module Fluent
       end
 
       def check_slow_flush(start)
-        elapsed_time = Process.clock_gettime(PROCESS_CLOCK_ID) - start
+        elapsed_time = Fluent::Clock.now - start
         if elapsed_time > @slow_flush_log_threshold
           log.warn "buffer flush took longer time than slow_flush_log_threshold:",
                    elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
@@ -1217,7 +1226,7 @@ module Fluent
       def flush_thread_run(state)
         flush_thread_interval = @buffer_config.flush_thread_interval
 
-        state.next_clock = Process.clock_gettime(PROCESS_CLOCK_ID) + flush_thread_interval
+        state.next_clock = Fluent::Clock.now + flush_thread_interval
 
         while !self.after_started? && !self.stopped?
           sleep 0.5
@@ -1227,7 +1236,7 @@ module Fluent
         begin
           # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase
           while @output_flush_threads_running
-            current_clock = Process.clock_gettime(PROCESS_CLOCK_ID)
+            current_clock = Fluent::Clock.now
             interval = state.next_clock - current_clock
 
             if state.next_clock <= current_clock && (!@retry || @retry_mutex.synchronize{ @retry.next_time } <= Time.now)
@@ -1238,7 +1247,7 @@ module Fluent
               # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
               #       because @retry still exists (#commit_write is not called yet in #try_flush)
               #       @retry should be cleared if delayed commit is enabled? Or any other solution?
-              state.next_clock = Process.clock_gettime(PROCESS_CLOCK_ID) + interval
+              state.next_clock = Fluent::Clock.now + interval
             end
 
             if @dequeued_chunks_mutex.synchronize{ !@dequeued_chunks.empty? && @dequeued_chunks.first.expired? }

data/lib/fluent/plugin/storage_local.rb

@@ -42,6 +42,7 @@ module Fluent
       def initialize
         super
         @store = {}
+        @multi_workers_available = nil
       end
 
       def configure(conf)
@@ -49,9 +50,23 @@ module Fluent
 
         @on_memory = false
         if @path
-          # use it
+          if File.exist?(@path) && File.file?(@path)
+            @multi_workers_available = false
+          elsif File.exist?(@path) && File.directory?(@path)
+            @path = File.join(@path, "worker#{fluentd_worker_id}", "storage.json")
+            @multi_workers_available = true
+          else # path file/directory doesn't exist
+            if @path.end_with?('.json') # file
+              @multi_workers_available = false
+            else # directory
+              @path = File.join(@path, "worker#{fluentd_worker_id}", "storage.json")
+              @multi_workers_available = true
+            end
+          end
         elsif root_dir = owner.plugin_root_dir
-          @path = File.join(root_dir, 'storage.json')
+          basename = (conf.arg && !conf.arg.empty?) ? "storage.#{conf.arg}.json" : "storage.json"
+          @path = File.join(root_dir, basename)
+          @multi_workers_available = true
         else
           if @persistent
             raise Fluent::ConfigError, "Plugin @id or path for <storage> required when 'persistent' is true"
@@ -62,6 +77,7 @@ module Fluent
               log.info "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
             end
             @on_memory = true
+            @multi_workers_available = true
           end
         end
 
@@ -83,6 +99,13 @@ module Fluent
         end
       end
 
+      def multi_workers_ready?
+        unless @multi_workers_available
+          log.error "local plugin storage with multi workers should be configured to use directory 'path', or system root_dir and plugin id"
+        end
+        @multi_workers_available
+      end
+
       def load
         return if @on_memory
         return unless File.exist?(@path)

data/lib/fluent/plugin_helper/cert_option.rb

@@ -0,0 +1,159 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'openssl'
+require 'socket'
+
+# this module is only for Socket/Server plugin helpers
+module Fluent
+  module PluginHelper
+    module CertOption
+      def cert_option_create_context(version, insecure, ciphers, conf)
+        cert, key, extra = cert_option_server_validate!(conf)
+
+        ctx = OpenSSL::SSL::SSLContext.new(version)
+        unless insecure
+          # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+          # https://bugs.ruby-lang.org/issues/9424
+          ctx.set_params({})
+
+          ctx.ciphers = ciphers
+        end
+
+        ctx.cert = cert
+        ctx.key = key
+        if extra && !extra.empty?
+          ctx.extra_chain_cert = extra
+        end
+
+        ctx
+      end
+
+      def cert_option_server_validate!(conf)
+        case
+        when conf.cert_path
+          raise Fluent::ConfigError, "private_key_path is required when cert_path is specified" unless conf.private_key_path
+          raise Fluent::ConfigError, "private_key_passphrase is required when cert_path is specified" unless conf.private_key_passphrase
+          cert_option_load(conf.cert_path, conf.private_key_path, conf.private_key_passphrase)
+
+        when conf.ca_cert_path
+          raise Fluent::ConfigError, "ca_private_key_path is required when ca_cert_path is specified" unless conf.ca_private_key_path
+          raise Fluent::ConfigError, "ca_private_key_passphrase is required when ca_cert_path is specified" unless conf.ca_private_key_passphrase
+          generate_opts = cert_option_cert_generation_opts_from_conf(conf)
+          cert_option_generate_server_pair_by_ca(
+            conf.ca_cert_path,
+            conf.ca_private_key_path,
+            conf.ca_private_key_passphrase,
+            generate_opts
+          )
+
+        when conf.insecure
+          log.warn "insecure TLS communication server is configured (using 'insecure' mode)"
+          generate_opts = cert_option_cert_generation_opts_from_conf(conf)
+          cert_option_generate_server_pair_self_signed(generate_opts)
+
+        else
+          raise Fluent::ConfigError, "no valid cert options configured. specify either 'cert_path', 'ca_cert_path' or 'insecure'"
+        end
+      end
+
+      def cert_option_load(cert_path, private_key_path, private_key_passphrase)
+        key = OpenSSL::PKey::RSA.new(File.read(private_key_path), private_key_passphrase)
+        certs = cert_option_certificates_from_file(cert_path)
+        cert = certs.shift
+        return cert, key, certs
+      end
+
+      def cert_option_cert_generation_opts_from_conf(conf)
+        {
+          private_key_length: conf.generate_private_key_length,
+          country: conf.generate_cert_country,
+          state: conf.generate_cert_state,
+          locality: conf.generate_cert_locality,
+          common_name: conf.generate_cert_common_name || ::Socket.gethostname,
+          expiration: conf.generate_cert_expiration,
+          digest: conf.generate_cert_digest,
+        }
+      end
+
+      def cert_option_generate_pair(opts, issuer = nil)
+        key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+        subject = OpenSSL::X509::Name.new
+        subject.add_entry('C', opts[:country])
+        subject.add_entry('ST', opts[:state])
+        subject.add_entry('L', opts[:locality])
+        subject.add_entry('CN', opts[:common_name])
+
+        issuer ||= subject
+
+        cert = OpenSSL::X509::Certificate.new
+        cert.not_before = Time.at(0)
+        cert.not_after = Time.now + opts[:expiration]
+        cert.public_key = key
+        cert.serial = 1
+        cert.issuer = issuer
+        cert.subject  = subject
+
+        return cert, key
+      end
+
+      def cert_option_generate_ca_pair_self_signed(generate_opts)
+        cert, key = cert_option_generate_pair(generate_opts)
+
+        # basicConstraints: this cert is for CA or not
+        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
+
+        cert.sign(key, generate_opts[:digest].to_s)
+        return cert, key
+      end
+
+      def cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, generate_opts)
+        ca_key = OpenSSL::PKey::RSA.new(File.read(ca_key_path), ca_key_passphrase)
+        ca_cert = OpenSSL::X509::Certificate.new(File.read(ca_cert_path))
+        cert, key = cert_option_generate_pair(generate_opts, ca_cert.subject)
+        raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
+
+        # basicConstraints: this cert is for CA or not
+        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
+        cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+
+        cert.sign(ca_key, generate_opts[:digest].to_s)
+        return cert, key, nil
+      end
+
+      def cert_option_generate_server_pair_self_signed(generate_opts)
+        cert, key = cert_option_generate_pair(generate_opts)
+        raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
+
+        # basicConstraints: this cert is for CA or not
+        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
+        cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+
+        cert.sign(key, generate_opts[:digest].to_s)
+        return cert, key, nil
+      end
+
+      def cert_option_certificates_from_file(path)
+        data = File.read(path)
+        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n', Regexp::MULTILINE)
+        list = []
+        data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
+        list
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/child_process.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/plugin_helper/thread'
 require 'fluent/plugin_helper/timer'
+require 'fluent/clock'
 
 require 'open3'
 require 'timeout'
@@ -125,7 +126,6 @@ module Fluent
         @_child_process_kill_timeout = CHILD_PROCESS_DEFAULT_KILL_TIMEOUT
         @_child_process_mutex = Mutex.new
         @_child_process_processes = {} # pid => ProcessInfo
-        @_child_process_clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
       end
 
       def stop
@@ -154,8 +154,8 @@ module Fluent
           child_process_kill(process_info)
         end
 
-        exit_wait_timeout = Process.clock_gettime(@_child_process_clock_id) + @_child_process_exit_timeout
-        while Process.clock_gettime(@_child_process_clock_id) < exit_wait_timeout
+        exit_wait_timeout = Fluent::Clock.now + @_child_process_exit_timeout
+        while Fluent::Clock.now < exit_wait_timeout
           process_exists = false
           @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
             unless @_child_process_processes[pid].exit_status
@@ -183,9 +183,9 @@ module Fluent
 
             living_process_exist = true
 
-            process_info.killed_at ||= Process.clock_gettime(@_child_process_clock_id) # for illegular case (e.g., created after shutdown)
+            process_info.killed_at ||= Fluent::Clock.now # for illegular case (e.g., created after shutdown)
             timeout_at = process_info.killed_at + @_child_process_kill_timeout
-            now = Process.clock_gettime(@_child_process_clock_id)
+            now = Fluent::Clock.now
             next if now < timeout_at
 
             child_process_kill(process_info, force: true)
@@ -207,7 +207,7 @@ module Fluent
 
       def child_process_kill(pinfo, force: false)
         return if !pinfo
-        pinfo.killed_at = Process.clock_gettime(@_child_process_clock_id) unless force
+        pinfo.killed_at = Fluent::Clock.now unless force
 
         pid = pinfo.pid
         begin