dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/plugin/out_http.rb

@@ -0,0 +1,280 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'net/http'
+require 'uri'
+require 'openssl'
+require 'fluent/tls'
+require 'fluent/plugin/output'
+require 'fluent/plugin_helper/socket'
+
+# patch Net::HTTP to support extra_chain_cert which was added in Ruby feature #9758.
+# see: https://github.com/ruby/ruby/commit/31af0dafba6d3769d2a39617c0dddedb97883712
+unless Net::HTTP::SSL_IVNAMES.include?(:@extra_chain_cert)
+  class Net::HTTP
+    SSL_IVNAMES << :@extra_chain_cert
+    SSL_ATTRIBUTES << :extra_chain_cert
+    attr_accessor :extra_chain_cert
+  end
+end
+
+module Fluent::Plugin
+  class HTTPOutput < Output
+    Fluent::Plugin.register_output('http', self)
+
+    class RetryableResponse < StandardError; end
+
+    helpers :formatter
+
+    desc 'The endpoint for HTTP request, e.g. http://example.com/api'
+    config_param :endpoint, :string
+    desc 'The method for HTTP request'
+    config_param :http_method, :enum, list: [:put, :post], default: :post
+    desc 'The proxy for HTTP request'
+    config_param :proxy, :string, default: ENV['HTTP_PROXY'] || ENV['http_proxy']
+    desc 'Content-Type for HTTP request'
+    config_param :content_type, :string, default: nil
+    desc 'JSON array data format for HTTP request body'
+    config_param :json_array, :bool, default: false
+    desc 'Additional headers for HTTP request'
+    config_param :headers, :hash, default: nil
+    desc 'Additional placeholder based headers for HTTP request'
+    config_param :headers_from_placeholders, :hash, default: nil
+
+    desc 'The connection open timeout in seconds'
+    config_param :open_timeout, :integer, default: nil
+    desc 'The read timeout in seconds'
+    config_param :read_timeout, :integer, default: nil
+    desc 'The TLS timeout in seconds'
+    config_param :ssl_timeout, :integer, default: nil
+
+    desc 'The CA certificate path for TLS'
+    config_param :tls_ca_cert_path, :string, default: nil
+    desc 'The client certificate path for TLS'
+    config_param :tls_client_cert_path, :string, default: nil
+    desc 'The client private key path for TLS'
+    config_param :tls_private_key_path, :string, default: nil
+    desc 'The client private key passphrase for TLS'
+    config_param :tls_private_key_passphrase, :string, default: nil, secret: true
+    desc 'The verify mode of TLS'
+    config_param :tls_verify_mode, :enum, list: [:none, :peer], default: :peer
+    desc 'The default version of TLS'
+    config_param :tls_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
+    desc 'The cipher configuration of TLS'
+    config_param :tls_ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT
+
+    desc 'Raise UnrecoverableError when the response is non success, 4xx/5xx'
+    config_param :error_response_as_unrecoverable, :bool, default: true
+    desc 'The list of retryable response code'
+    config_param :retryable_response_codes, :array, value_type: :integer, default: nil
+
+    config_section :format do
+      config_set_default :@type, 'json'
+    end
+
+    config_section :auth, required: false, multi: false do
+      desc 'The method for HTTP authentication'
+      config_param :method, :enum, list: [:basic], default: :basic
+      desc 'The username for basic authentication'
+      config_param :username, :string, default: nil
+      desc 'The password for basic authentication'
+      config_param :password, :string, default: nil, secret: true
+    end
+
+    def initialize
+      super
+
+      @uri = nil
+      @proxy_uri = nil
+      @formatter = nil
+    end
+
+    def configure(conf)
+      super
+
+      if @retryable_response_codes.nil?
+        log.warn('Status code 503 is going to be removed from default `retryable_response_codes` from fluentd v2. Please add it by yourself if you wish')
+        @retryable_response_codes = [503]
+      end
+
+      @http_opt = setup_http_option
+      @proxy_uri = URI.parse(@proxy) if @proxy
+      @formatter = formatter_create
+      @content_type = setup_content_type unless @content_type
+
+      if @json_array
+        if @formatter_configs.first[:@type] != "json"
+          raise Fluent::ConfigError, "json_array option could be used with json formatter only"
+        end
+        define_singleton_method(:format, method(:format_json_array))
+      end
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def formatted_to_msgpack_binary?
+      @formatter_configs.first[:@type] == 'msgpack'
+    end
+
+    def format(tag, time, record)
+      @formatter.format(tag, time, record)
+    end
+
+    def format_json_array(tag, time, record)
+      @formatter.format(tag, time, record) << ","
+    end
+
+    def write(chunk)
+      uri = parse_endpoint(chunk)
+      req = create_request(chunk, uri)
+
+      log.debug { "#{@http_method.capitalize} data to #{uri.to_s} with chunk(#{dump_unique_id_hex(chunk.unique_id)})" }
+
+      send_request(uri, req)
+    end
+
+    private
+
+    def setup_content_type
+      case @formatter_configs.first[:@type]
+      when 'json'
+        @json_array ? 'application/json' : 'application/x-ndjson'
+      when 'csv'
+        'text/csv'
+      when 'tsv', 'ltsv'
+        'text/tab-separated-values'
+      when 'msgpack'
+        'application/x-msgpack'
+      when 'out_file', 'single_value', 'stdout', 'hash'
+        'text/plain'
+      else
+        raise Fluent::ConfigError, "can't determine Content-Type from formatter type. Set content_type parameter explicitly"
+      end
+    end
+
+    def setup_http_option
+      use_ssl = @endpoint.start_with?('https')
+      opt = {
+        open_timeout: @open_timeout,
+        read_timeout: @read_timeout,
+        ssl_timeout: @ssl_timeout,
+        use_ssl: use_ssl
+      }
+
+      if use_ssl
+        if @tls_ca_cert_path
+          raise Fluent::ConfigError, "tls_ca_cert_path is wrong: #{@tls_ca_cert_path}" unless File.file?(@tls_ca_cert_path)
+          opt[:ca_file] = @tls_ca_cert_path
+        end
+        if @tls_client_cert_path
+          raise Fluent::ConfigError, "tls_client_cert_path is wrong: #{@tls_client_cert_path}" unless File.file?(@tls_client_cert_path)
+
+          bundle = File.read(@tls_client_cert_path)
+          bundle_certs = bundle.scan(/-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/)
+          opt[:cert] = OpenSSL::X509::Certificate.new(bundle_certs[0])
+
+          intermediate_certs = bundle_certs[1..-1]
+          if intermediate_certs
+            opt[:extra_chain_cert] = intermediate_certs.map { |cert| OpenSSL::X509::Certificate.new(cert) }
+          end
+        end
+        if @tls_private_key_path
+          raise Fluent::ConfigError, "tls_private_key_path is wrong: #{@tls_private_key_path}" unless File.file?(@tls_private_key_path)
+          opt[:key] = OpenSSL::PKey.read(File.read(@tls_private_key_path), @tls_private_key_passphrase)
+        end
+        opt[:verify_mode] = case @tls_verify_mode
+                            when :none
+                              OpenSSL::SSL::VERIFY_NONE
+                            when :peer
+                              OpenSSL::SSL::VERIFY_PEER
+                            end
+        opt[:ciphers] = @tls_ciphers
+        opt[:ssl_version] = @tls_version
+      end
+
+      opt
+    end
+
+    def parse_endpoint(chunk)
+      endpoint = extract_placeholders(@endpoint, chunk)
+      URI.parse(endpoint)
+    end
+
+    def set_headers(req, chunk)
+      if @headers
+        @headers.each do |k, v|
+          req[k] = v
+        end
+      end
+      if @headers_from_placeholders
+        @headers_from_placeholders.each do |k, v|
+          req[k] = extract_placeholders(v, chunk)
+        end
+      end
+      req['Content-Type'] = @content_type
+    end
+
+    def create_request(chunk, uri)
+      req = case @http_method
+            when :post
+              Net::HTTP::Post.new(uri.request_uri)
+            when :put
+              Net::HTTP::Put.new(uri.request_uri)
+            end
+      if @auth
+        req.basic_auth(@auth.username, @auth.password)
+      end
+      set_headers(req, chunk)
+      if @json_minify
+        req.body = JSON.minify(req.body)
+        req.body = @json_array ? JSON.minify("[#{chunk.read.chop}]") : chunk.read
+      else
+        req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
+      end
+      req
+    end
+
+    def send_request(uri, req)
+      res = if @proxy_uri
+              Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt) { |http|
+                http.request(req)
+              }
+            else
+              Net::HTTP.start(uri.host, uri.port, @http_opt) { |http|
+                http.request(req)
+              }
+            end
+
+      if res.is_a?(Net::HTTPSuccess)
+        log.debug { "#{res.code} #{res.message.rstrip}#{res.body.lstrip}" }
+      else
+        msg = "#{res.code} #{res.message.rstrip} #{res.body.lstrip}"
+
+        if @retryable_response_codes.include?(res.code.to_i)
+          raise RetryableResponse, msg
+        end
+
+        if @error_response_as_unrecoverable
+          raise Fluent::UnrecoverableError, msg
+        else
+          log.error "got error response from '#{@http_method.capitalize} #{uri.to_s}' : #{msg}"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_null.rb

@@ -0,0 +1,74 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class NullOutput < Output
+    # This plugin is for tests of non-buffered/buffered plugins
+    Fluent::Plugin.register_output('null', self)
+
+    desc "The parameter for testing to simulate output plugin which never succeed to flush."
+    config_param :never_flush, :bool, default: false
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
+    def prefer_buffered_processing
+      false
+    end
+
+    def prefer_delayed_commit
+      @delayed
+    end
+
+    attr_accessor :feed_proc, :delayed
+
+    def initialize
+      super
+      @delayed = false
+      @feed_proc = nil
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def process(tag, es)
+      raise "failed to flush" if @never_flush
+      # Do nothing
+    end
+
+    def write(chunk)
+      raise "failed to flush" if @never_flush
+      if @feed_proc
+        @feed_proc.call(chunk)
+      end
+    end
+
+    def try_write(chunk)
+      raise "failed to flush" if @never_flush
+      if @feed_proc
+        @feed_proc.call(chunk)
+      end
+      # not to commit chunks for testing
+      # commit_write(chunk.unique_id)
+    end
+  end
+end

data/lib/fluent/plugin/out_relabel.rb

@@ -0,0 +1,32 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class RelabelOutput < Output
+    Fluent::Plugin.register_output('relabel', self)
+    helpers :event_emitter
+
+    def multi_workers_ready?
+      true
+    end
+
+    def process(tag, es)
+      router.emit_stream(tag, es)
+    end
+  end
+end

data/lib/fluent/plugin/out_roundrobin.rb

@@ -0,0 +1,84 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/multi_output'
+require 'fluent/config/error'
+
+module Fluent::Plugin
+  class RoundRobinOutput < MultiOutput
+    Fluent::Plugin.register_output('roundrobin', self)
+
+    config_section :store do
+      config_param :weight, :integer, default: 1
+    end
+
+    def initialize
+      super
+      @weights = []
+    end
+
+    attr_reader :weights
+
+    def configure(conf)
+      super
+
+      @stores.each do |store|
+        @weights << store.weight
+      end
+      @rr = -1  # starts from @output[0]
+      @rand_seed = Random.new.seed
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+      rebuild_weight_array
+    end
+
+    def process(tag, es)
+      next_output.emit_events(tag, es)
+    end
+
+    private
+
+    def next_output
+      @rr = 0 if (@rr += 1) >= @weight_array.size
+      @weight_array[@rr]
+    end
+
+    def rebuild_weight_array
+      gcd = @weights.inject(0) {|r,w| r.gcd(w) }
+
+      weight_array = []
+      @outputs.zip(@weights).each {|output,weight|
+        (weight / gcd).times {
+          weight_array << output
+        }
+      }
+
+      # don't randomize order if all weight is 1 (=default)
+      if @weights.any? {|w| w > 1 }
+        r = Random.new(@rand_seed)
+        weight_array.sort_by! { r.rand }
+      end
+
+      @weight_array = weight_array
+    end
+  end
+end

data/lib/fluent/plugin/out_secondary_file.rb

@@ -0,0 +1,131 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "fileutils"
+require "fluent/plugin/file_util"
+require "fluent/plugin/output"
+require "fluent/config/error"
+
+module Fluent::Plugin
+  class SecondaryFileOutput < Output
+    Fluent::Plugin.register_output("secondary_file", self)
+
+    PLACEHOLDER_REGEX = /\${(tag(\[\d+\])?|[\w.@-]+)}/
+
+    desc "The directory path of the output file."
+    config_param :directory, :string
+    desc "The basename of the output file."
+    config_param :basename, :string, default: "dump.bin"
+    desc "The flushed chunk is appended to existence file or not."
+    config_param :append, :bool, default: false
+    config_param :compress, :enum, list: [:text, :gzip], default: :text
+
+    def configure(conf)
+      super
+
+      unless @as_secondary
+        raise Fluent::ConfigError, "This plugin can only be used in the <secondary> section"
+      end
+
+      if @basename.include?("/")
+        raise Fluent::ConfigError, "basename should not include `/`"
+      end
+
+      @path_without_suffix = File.join(@directory, @basename)
+      validate_compatible_with_primary_buffer!(@path_without_suffix)
+
+      @suffix = case @compress
+                when :text
+                  ""
+                when :gzip
+                  ".gz"
+                end
+
+      test_path = @path_without_suffix
+      unless Fluent::FileUtil.writable_p?(test_path)
+        raise Fluent::ConfigError, "out_secondary_file: `#{@directory}` should be writable"
+      end
+
+      @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
+      @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
+    end
+
+    def multi_workers_ready?
+      ### TODO: add hack to synchronize for multi workers
+      true
+    end
+
+    def write(chunk)
+      path_without_suffix = extract_placeholders(@path_without_suffix, chunk)
+      path = generate_path(path_without_suffix)
+      FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
+
+      case @compress
+      when :text
+        File.open(path, "ab", @file_perm) {|f|
+          f.flock(File::LOCK_EX)
+          chunk.write_to(f)
+        }
+      when :gzip
+        File.open(path, "ab", @file_perm) {|f|
+          f.flock(File::LOCK_EX)
+          gz = Zlib::GzipWriter.new(f)
+          chunk.write_to(gz)
+          gz.close
+        }
+      end
+
+      path
+    end
+
+    private
+
+    def validate_compatible_with_primary_buffer!(path_without_suffix)
+      placeholders = path_without_suffix.scan(PLACEHOLDER_REGEX).flat_map(&:first) # to trim suffix [\d+]
+
+      if !@chunk_key_time && has_time_format?(path_without_suffix)
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder, remove time formats, like `%Y%m%d`, from basename or directory"
+      end
+
+      if !@chunk_key_tag && (ph = placeholders.find { |placeholder| placeholder.match(/tag(\[\d+\])?/) })
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove tag placeholder, like `${tag}`, from basename or directory"
+      end
+
+      vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) || (placeholder == 'chunk_id') }
+
+      if ph = vars.find { |v| !@chunk_keys.include?(v) }
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove variable placeholder, like `${varname}`, from basename or directory"
+      end
+    end
+
+    def has_time_format?(str)
+      str != Time.now.strftime(str)
+    end
+
+    def generate_path(path_without_suffix)
+      if @append
+        "#{path_without_suffix}#{@suffix}"
+      else
+        i = 0
+        loop do
+          path = "#{path_without_suffix}.#{i}#{@suffix}"
+          return path unless File.exist?(path)
+          i += 1
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_stdout.rb

@@ -0,0 +1,74 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class StdoutOutput < Output
+    Fluent::Plugin.register_output('stdout', self)
+
+    helpers :inject, :formatter, :compat_parameters
+
+    DEFAULT_LINE_FORMAT_TYPE = 'stdout'
+    DEFAULT_FORMAT_TYPE = 'json'
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_LINE_FORMAT_TYPE
+      config_set_default :output_type, DEFAULT_FORMAT_TYPE
+    end
+
+    def prefer_buffered_processing
+      false
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    attr_accessor :formatter
+
+    def configure(conf)
+      compat_parameters_convert(conf, :inject, :formatter)
+
+      super
+
+      @formatter = formatter_create
+    end
+
+    def process(tag, es)
+      es = inject_values_to_event_stream(tag, es)
+      es.each {|time,record|
+        $log.write(format(tag, time, record))
+      }
+      $log.flush
+    end
+
+    def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
+      @formatter.format(tag, time, record).chomp + "\n"
+    end
+
+    def write(chunk)
+      chunk.write_to($log)
+    end
+  end
+end