fluent-plugin-elasticsearch 4.3.2 → 5.0.3

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.3.2'
+  s.version       = '5.0.3'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
 
 
   s.add_development_dependency 'rake', '>= 0'
+  s.add_development_dependency 'webrick', '~> 1.7.0'
   s.add_development_dependency 'webmock', '~> 3'
   s.add_development_dependency 'test-unit', '~> 3.3.0'
   s.add_development_dependency 'minitest', '~> 5.8'

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -13,6 +13,7 @@ begin
   require 'strptime'
 rescue LoadError
 end
+require 'resolv'
 
 require 'fluent/plugin/output'
 require 'fluent/event'
@@ -493,7 +494,12 @@ EOC
 
     def detect_es_major_version
       @_es_info ||= client.info
-      unless version = @_es_info.dig("version", "number")
+      begin
+        unless version = @_es_info.dig("version", "number")
+          version = @default_elasticsearch_version
+        end
+      rescue NoMethodError => e
+        log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
         version = @default_elasticsearch_version
       end
       version.to_i
@@ -663,7 +669,11 @@ EOC
           end
         end.compact
       else
-        [{host: @host, port: @port, scheme: @scheme.to_s}]
+        if Resolv::IPv6::Regex.match(@host)
+          [{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
+        else
+          [{host: @host, port: @port, scheme: @scheme.to_s}]
+        end
       end.each do |host|
         host.merge!(user: @user, password: @password) if !host[:user] && @user
         host.merge!(path: @path) if !host[:path] && @path

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -0,0 +1,218 @@
+require_relative 'out_elasticsearch'
+
+module Fluent::Plugin
+  class ElasticsearchOutputDataStream < ElasticsearchOutput
+
+    Fluent::Plugin.register_output('elasticsearch_data_stream', self)
+
+    helpers :event_emitter
+
+    config_param :data_stream_name, :string
+    # Elasticsearch 7.9 or later always support new style of index template.
+    config_set_default :use_legacy_template, false
+
+    INVALID_START_CHRACTERS = ["-", "_", "+", "."]
+    INVALID_CHARACTERS = ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"]
+
+    def configure(conf)
+      super
+
+      begin
+        require 'elasticsearch/api'
+        require 'elasticsearch/xpack'
+      rescue LoadError
+        raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
+      end
+
+      # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
+      unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
+        validate_data_stream_name
+      else
+        @use_placeholder = true
+        @data_stream_names = []
+      end
+
+      @client = client
+      unless @use_placeholder
+        begin
+          @data_stream_names = [@data_stream_name]
+          create_ilm_policy(@data_stream_name)
+          create_index_template(@data_stream_name)
+          create_data_stream(@data_stream_name)
+        rescue => e
+          raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
+        end
+      end
+    end
+
+    def validate_data_stream_name
+      unless valid_data_stream_name?
+        unless start_with_valid_characters?
+          if not_dots?
+            raise Fluent::ConfigError, "'data_stream_name' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{@data_stream_name}>"
+          else
+            raise Fluent::ConfigError, "'data_stream_name' must not be . or ..: <#{@data_stream_name}>"
+          end
+        end
+        unless valid_characters?
+          raise Fluent::ConfigError, "'data_stream_name' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{@data_stream_name}>"
+        end
+        unless lowercase_only?
+          raise Fluent::ConfigError, "'data_stream_name' must be lowercase only: <#{@data_stream_name}>"
+        end
+        if @data_stream_name.bytes.size > 255
+          raise Fluent::ConfigError, "'data_stream_name' must not be longer than 255 bytes: <#{@data_stream_name}>"
+        end
+      end
+    end
+
+    def create_ilm_policy(name)
+      return if data_stream_exist?(name)
+      params = {
+        policy_id: "#{name}_policy",
+        body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
+      }
+      retry_operate(@max_retry_putting_template,
+                    @fail_on_putting_template_retry_exceed,
+                    @catch_transport_exception_on_retry) do
+        @client.xpack.ilm.put_policy(params)
+      end
+    end
+
+    def create_index_template(name)
+      return if data_stream_exist?(name)
+      body = {
+        "index_patterns" => ["#{name}*"],
+        "data_stream" => {},
+        "template" => {
+          "settings" => {
+            "index.lifecycle.name" => "#{name}_policy"
+          }
+        }
+      }
+      params = {
+        name: name,
+        body: body
+      }
+      retry_operate(@max_retry_putting_template,
+                    @fail_on_putting_template_retry_exceed,
+                    @catch_transport_exception_on_retry) do
+        @client.indices.put_index_template(params)
+      end
+    end
+
+    def data_stream_exist?(name)
+      params = {
+        "name": name
+      }
+      begin
+        response = @client.indices.get_data_stream(params)
+        return (not response.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound))
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+        log.info "Specified data stream does not exist. Will be created: <#{e}>"
+        return false
+      end
+    end
+
+    def create_data_stream(name)
+      return if data_stream_exist?(name)
+      params = {
+        "name": name
+      }
+      retry_operate(@max_retry_putting_template,
+                    @fail_on_putting_template_retry_exceed,
+                    @catch_transport_exception_on_retry) do
+        @client.indices.create_data_stream(params)
+      end
+    end
+
+    def valid_data_stream_name?
+      lowercase_only? and
+        valid_characters? and
+        start_with_valid_characters? and
+        not_dots? and
+        @data_stream_name.bytes.size <= 255
+    end
+
+    def lowercase_only?
+      @data_stream_name.downcase == @data_stream_name
+    end
+
+    def valid_characters?
+      not (INVALID_CHARACTERS.each.any? do |v| @data_stream_name.include?(v) end)
+    end
+
+    def start_with_valid_characters?
+      not (INVALID_START_CHRACTERS.each.any? do |v| @data_stream_name.start_with?(v) end)
+    end
+
+    def not_dots?
+      not (@data_stream_name == "." or @data_stream_name == "..")
+    end
+
+    def client_library_version
+      Elasticsearch::VERSION
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def write(chunk)
+      data_stream_name = @data_stream_name
+      if @use_placeholder
+        data_stream_name = extract_placeholders(@data_stream_name, chunk)
+        unless @data_stream_names.include?(data_stream_name)
+          begin
+            create_ilm_policy(data_stream_name)
+            create_index_template(data_stream_name)
+            create_data_stream(data_stream_name)
+            @data_stream_names << data_stream_name
+          rescue => e
+            raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
+          end
+        end
+      end
+
+      bulk_message = ""
+      headers = {
+        CREATE_OP => {}
+      }
+      tag = chunk.metadata.tag
+      chunk.msgpack_each do |time, record|
+        next unless record.is_a? Hash
+
+        begin
+          record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
+          bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
+      end
+
+      params = {
+        index: data_stream_name,
+        body: bulk_message
+      }
+      begin
+        response = @client.bulk(params)
+        if response['errors']
+          log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
+        end
+      rescue => e
+        log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{e.message}"
+      end
+    end
+
+    def append_record_to_messages(op, meta, header, record, msgs)
+      header[CREATE_OP] = meta
+      msgs << @dump_proc.call(header) << BODY_DELIMITER
+      msgs << @dump_proc.call(record) << BODY_DELIMITER
+      msgs
+    end
+
+    def retry_stream_retryable?
+      @buffer.storable?
+    end
+  end
+end

data/test/plugin/test_out_elasticsearch.rb

@@ -370,6 +370,65 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     }
   end
 
+  sub_test_case 'Check client.info response' do
+    def create_driver(conf='', es_version=5, client_version="\"5.0\"")
+      # For request stub to detect compatibility.
+      @client_version ||= client_version
+      @default_elasticsearch_version ||= es_version
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def detect_es_major_version
+          @_es_info ||= client.info
+          begin
+            unless version = @_es_info.dig("version", "number")
+              version = @default_elasticsearch_version
+            end
+          rescue NoMethodError => e
+            log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
+            version = @default_elasticsearch_version
+          end
+          version.to_i
+        end
+      CODE
+
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def client_library_version
+          #{@client_version}
+        end
+      CODE
+      @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
+        # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
+        if !defined?(Fluent::Plugin::Output)
+          def format(tag, time, record)
+            [time, record].to_msgpack
+          end
+        end
+      }.configure(conf)
+    end
+
+    def stub_elastic_info_bad(url="http://localhost:9200/", version="6.4.2")
+      body ="{\"version\":{\"number\":\"#{version}\"}}"
+      stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'text/plain' } })
+    end
+
+    test 'handle invalid client.info' do
+      stub_elastic_info_bad("https://logs.fluentd.com:24225/es//", "7.7.1")
+      config = %{
+        host     logs.fluentd.com
+        port     24225
+        scheme   https
+        path     /es/
+        user     john
+        password doe
+        default_elasticsearch_version 7
+        scheme https
+        @log_level info
+      }
+      d = create_driver(config, 7, "\"7.10.1\"")
+      logs = d.logs
+      assert_logs_include(logs, /can not dig version information. Assuming Elasticsearch 7/)
+    end
+  end
+
   sub_test_case 'Check TLS handshake stuck warning log' do
     test 'warning TLS log' do
       config = %{
@@ -3553,6 +3612,74 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
+  class IPv6AdressStringHostsTest < self
+    def test_legacy_hosts_list
+      config = %{
+        hosts    "[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:50,host2:100,host3"
+        scheme   https
+        path     /es/
+        port     123
+      }
+      instance = driver(config).instance
+
+      assert_raise(URI::InvalidURIError) do
+        instance.get_connection_options[:hosts].length
+      end
+    end
+
+    def test_hosts_list
+      config = %{
+        hosts    https://john:password@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
+        path     /default_path
+        user     default_user
+        password default_password
+      }
+      instance = driver(config).instance
+
+      assert_equal 2, instance.get_connection_options[:hosts].length
+      host1, host2 = instance.get_connection_options[:hosts]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 443, host1[:port]
+      assert_equal 'https', host1[:scheme]
+      assert_equal 'john', host1[:user]
+      assert_equal 'password', host1[:password]
+      assert_equal '/elastic/', host1[:path]
+
+      assert_equal 'host2', host2[:host]
+      assert_equal 'http', host2[:scheme]
+      assert_equal 'default_user', host2[:user]
+      assert_equal 'default_password', host2[:password]
+      assert_equal '/default_path', host2[:path]
+    end
+
+    def test_hosts_list_with_escape_placeholders
+      config = %{
+        hosts    https://%{j+hn}:%{passw@rd}@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
+        path     /default_path
+        user     default_user
+        password default_password
+      }
+      instance = driver(config).instance
+
+      assert_equal 2, instance.get_connection_options[:hosts].length
+      host1, host2 = instance.get_connection_options[:hosts]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 443, host1[:port]
+      assert_equal 'https', host1[:scheme]
+      assert_equal 'j%2Bhn', host1[:user]
+      assert_equal 'passw%40rd', host1[:password]
+      assert_equal '/elastic/', host1[:path]
+
+      assert_equal 'host2', host2[:host]
+      assert_equal 'http', host2[:scheme]
+      assert_equal 'default_user', host2[:user]
+      assert_equal 'default_password', host2[:password]
+      assert_equal '/default_path', host2[:path]
+    end
+  end
+
   def test_single_host_params_and_defaults
     config = %{
       host     logs.google.com
@@ -3606,6 +3733,46 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert(ports.none? { |p| p == 9200 })
   end
 
+  class IPv6AdressStringHostTest < self
+    def test_single_host_params_and_defaults
+      config = %{
+        host     2404:7a80:d440:3000:192a:a292:bd7f:ca19
+        user     john
+        password doe
+      }
+      instance = driver(config).instance
+
+      assert_equal 1, instance.get_connection_options[:hosts].length
+      host1 = instance.get_connection_options[:hosts][0]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 9200, host1[:port]
+      assert_equal 'http', host1[:scheme]
+      assert_equal 'john', host1[:user]
+      assert_equal 'doe', host1[:password]
+      assert_equal nil, host1[:path]
+    end
+
+    def test_single_host_params_and_defaults_with_escape_placeholders
+      config = %{
+        host     2404:7a80:d440:3000:192a:a292:bd7f:ca19
+        user     %{j+hn}
+        password %{d@e}
+      }
+      instance = driver(config).instance
+
+      assert_equal 1, instance.get_connection_options[:hosts].length
+      host1 = instance.get_connection_options[:hosts][0]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 9200, host1[:port]
+      assert_equal 'http', host1[:scheme]
+      assert_equal 'j%2Bhn', host1[:user]
+      assert_equal 'd%40e', host1[:password]
+      assert_equal nil, host1[:path]
+    end
+  end
+
   def test_password_is_required_if_specify_user
     config = %{
       user john