fluent-plugin-elasticsearch 4.0.4 → 4.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4253b332821c59318a743dccc063cfe0bf5b6f5302727d0bf974280b41336ea
-  data.tar.gz: 8731c0ba69bb200b1c397fc87ad0237c16ca8f5a825d37d16c0844664afd6292
+  metadata.gz: 81cf6831267be2a7250ab5365cf783c0680e5065ff3e3df068707ab3528bdbfb
+  data.tar.gz: ea99a64f901f9daa9cbf1ac43bc51064048765872b9c1d29a3029dd9b9f56f24
 SHA512:
-  metadata.gz: 41eff72fc9b79075b3e3c2812c43fc34d4e789ca005bbe0d722e6d75ef9d01706841bba7214495a1d1bcdc3eeb5d86008c002e4aa4686508b22f1caafce731a3
-  data.tar.gz: 3087f4fa5c6c008092366652ee388690bd012c6528ab01eb699fe352760a584500c83ea903d22b40693bf3d141bf7ed220f491cd64be3be08fa9a7ca2308f246
+  metadata.gz: 6c750270cf3051fe974ebaad5063c82cd8eb9be3afc89326ce99cd5929890f2330b2397ef2fa683f1a060f035db2266862666e09282b60ef8102cc497d2a888b
+  data.tar.gz: bee1bc70e85c8c475d943abb922531c5e668bcd7457692d006cff1fb59a669845ce8ad4a1e6a45cdca58e4a0bc63626638eeef50714bf67d6ff1c7f1a1c3c5fb

data/.travis.yml

@@ -1,17 +1,39 @@
 language: ruby
 
-matrix:
+jobs:
   include:
     - rvm: 2.4.6
       gemfile: Gemfile
+      os: linux
+      arch: amd64
     - rvm: 2.5.5
       gemfile: Gemfile
+      os: linux
+      arch: amd64
     - rvm: 2.6.3
       gemfile: gemfiles/Gemfile.ilm
+      os: linux
+      arch: amd64
+    - rvm: 2.6.3
+      gemfile: gemfiles/Gemfile.elasticsearch.v6
+      os: linux
+      arch: amd64
+    - rvm: 2.6.3
+      gemfile: Gemfile
+      os: linux
+      arch: amd64
+    - rvm: 2.6.3
+      gemfile: Gemfile
+      os: linux
+      arch: arm64
     - rvm: 2.6.3
       gemfile: Gemfile
+      os: osx
+      osx_image: xcode11.3
     - rvm: 2.7.0
       gemfile: Gemfile
+      os: linux
+      arch: amd64
 
 gemfile:
  - Gemfile
@@ -20,4 +42,3 @@ before_install:
   - gem update --system=2.7.8
 
 script: bundle exec rake test
-sudo: false

data/Gemfile

@@ -7,3 +7,4 @@ gem 'simplecov', require: false
 gem 'coveralls', require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
+gem "oj"

data/History.md

@@ -2,6 +2,13 @@
 
 ### [Unreleased]
 
+### 4.0.5
+-  add logstash_dateformat as placeholder (#718)
+- Tweak travis.yml for suppressing validator warnings and add CI for Linux Arm64 architecture and macOS 10.14 (#724)
+- Elasticsearch ruby v7.5 (#723)
+- Add Oj serializer testcases for all job (#722)
+- Update documentation for ILM (#721)
+
 ### 4.0.4
 - Provide clearing caches timer (#719)
 

data/README.md

@@ -1639,17 +1639,31 @@ Main ILM feature parameters are:
 * `ilm_policy_id`
 * `ilm_policy`
 
+* Advanced usage parameters
+  * `application_name`
+  * `index_separator`
+
 They are not all mandatory parameters but they are used for ILM feature in effect.
 
 ILM target index alias is created with `index_name` or an index which is calculated from `logstash_prefix`.
 
 From Elasticsearch plugin v4.0.0, ILM target index will be calculated from `index_name` (normal mode) or `logstash_prefix` (using with `logstash_format`as true).
 
-When using `deflectoe_alias` parameter, Elasticsearch plugin will create ILM target indices alias with `deflector_alias` instead of `index_name` or an index which is calculated from `logstash_prefix`. This behavior should be kept due to backward ILM feature compatibility.
+When using `deflector_alias` parameter, Elasticsearch plugin will create ILM target indices alias with `deflector_alias` instead of `index_name` or an index which is calculated from `logstash_prefix`. This behavior should be kept due to backward ILM feature compatibility.
 
 And also, ILM feature users should specify their Elasticsearch template for ILM enabled indices.
 Because ILM settings are injected into their Elasticsearch templates.
 
+`application_name` and `index_separator` also affect alias index names.
+
+But this parameter is prepared for advanced usage.
+
+It usually should be used with default value which is `default`.
+
+Then, ILM parameters are used in alias index like as:
+
+`<index_name/logstash_prefix><index_separator><application_name>-000001`.
+
 #### Example ILM settings
 
 ```aconf

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.4'
+  s.version       = '4.0.5'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/gemfiles/Gemfile.elasticsearch.v6

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
+gemspec :path => "../"
+
+gem 'simplecov', require: false
+gem 'coveralls', require: false
+gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
+gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
+gem "elasticsearch", "~> 6.8.1"
+gem "elasticsearch-xpack"
+gem "oj"

data/gemfiles/Gemfile.ilm

@@ -8,3 +8,4 @@ gem 'coveralls', require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
 gem "elasticsearch-xpack"
+gem "oj"

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -358,9 +358,7 @@ EOC
         end
       end
 
-      version_arr = ::Elasticsearch::Transport::VERSION.split('.')
-
-      if (version_arr[0].to_i < 7) || (version_arr[0].to_i == 7 && version_arr[1].to_i < 2)
+      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
         if compression
           raise Fluent::ConfigError, <<-EOC
             Cannot use compression with elasticsearch-transport plugin version < 7.2.0
@@ -669,6 +667,7 @@ EOC
 
     def expand_placeholders(chunk)
       logstash_prefix = extract_placeholders(@logstash_prefix, chunk)
+      logstash_dateformat = extract_placeholders(@logstash_dateformat, chunk)
       index_name = extract_placeholders(@index_name, chunk)
       if @type_name
         type_name = extract_placeholders(@type_name, chunk)
@@ -700,7 +699,7 @@ EOC
       else
         pipeline = nil
       end
-      return logstash_prefix, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline
+      return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline
     end
 
     def multi_workers_ready?
@@ -774,7 +773,7 @@ EOC
     end
 
     def process_message(tag, meta, header, time, record, extracted_values)
-      logstash_prefix, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
+      logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
 
       if @flatten_hashes
         record = flatten_record(record)
@@ -800,7 +799,7 @@ EOC
         target_index = target_index_parent.delete(target_index_child_key)
       elsif @logstash_format
         dt = dt.new_offset(0) if @utc_index
-        target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
+        target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
       else
         target_index = index_name
       end
@@ -880,6 +879,7 @@ EOC
         placeholder?(:template_name, @template_name.to_s) ||
         @customize_template&.values&.any? { |value| placeholder?(:customize_template, value.to_s) } ||
         placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
+        placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
         placeholder?(:deflector_alias, @deflector_alias.to_s) ||
         placeholder?(:application_name, @application_name.to_s)
       log.debug("Need substitution: #{need_substitution}")
@@ -914,7 +914,7 @@ EOC
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
     def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
-      logstash_prefix, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
+      logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
       if deflector_alias
         template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
       else

data/test/plugin/test_oj_serializer.rb

@@ -0,0 +1,19 @@
+require_relative '../helper'
+require 'elasticsearch'
+
+class OjSerializerTest < Test::Unit::TestCase
+  def setup
+    begin
+      require 'fluent/plugin/oj_serializer'
+    rescue LoadError
+      omit "OjSerializer testcase needs oj gem."
+    end
+    @serializer = Fluent::Plugin::Serializer::Oj.new
+  end
+
+  def test_serializer
+    data = {"message" => "Hi"}
+    assert_equal data.to_json, @serializer.dump(data)
+    assert_equal data, @serializer.load(data.to_json)
+  end
+end

data/test/plugin/test_out_elasticsearch.rb

@@ -201,6 +201,14 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_compare(exp_matches, operator, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
   end
 
+  def alias_endpoint
+    if Gem::Version.create(::Elasticsearch::VERSION) >= Gem::Version.create("7.5.0-pre")
+      "_aliases"
+    else
+      "_alias"
+    end
+  end
+
   def test_configure
     config = %{
       host     logs.google.com
@@ -246,6 +254,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   test 'configure compression' do
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+
     config = %{
       compression_level best_compression
     }
@@ -255,6 +265,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   test 'check compression strategy' do
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+
     config = %{
       compression_level best_speed
     }
@@ -264,6 +276,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   test 'check content-encoding header with compression' do
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+
     config = %{
       compression_level best_compression
     }
@@ -273,6 +287,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   test 'check compression option is passed to transport' do
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+
     config = %{
       compression_level best_compression
     }
@@ -338,7 +354,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
         to_return(status: 200, body: "", headers: {})
       stub_request(:head, "http://localhost:9200/_alias/fluentd").
         to_return(status: 404, body: "", headers: {})
-      stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/_alias/fluentd").
+      stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/#{alias_endpoint}/fluentd").
         with(body: "{\"aliases\":{\"fluentd\":{\"is_write_index\":true}}}").
         to_return(status: 200, body: "", headers: {})
       stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E").
@@ -372,7 +388,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
         to_return(status: 200, body: "", headers: {})
       stub_request(:head, "http://localhost:9200/_alias/fluentd").
         to_return(status: 404, body: "", headers: {})
-      stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/_alias/fluentd").
+      stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/#{alias_endpoint}/fluentd").
         with(body: "{\"aliases\":{\"fluentd\":{\"is_write_index\":true}}}").
         to_return(status: 200, body: "", headers: {})
       stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E").
@@ -791,7 +807,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E").
       with(basic_auth: ['john', 'doe']).
       to_return(status: 200, body: "", headers: {})
-    stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E/_alias/myapp_deflector-test.template").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E/#{alias_endpoint}/myapp_deflector-test.template").
       with(basic_auth: ['john', 'doe'],
            body: "{\"aliases\":{\"myapp_deflector-test.template\":{\"is_write_index\":true}}}").
       to_return(:status => 200, :body => "", :headers => {})
@@ -849,7 +865,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E").
       with(basic_auth: ['john', 'doe']).
       to_return(status: 200, body: "", headers: {})
-    stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E/_alias/myapp_deflector-test.template").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E/#{alias_endpoint}/myapp_deflector-test.template").
       with(basic_auth: ['john', 'doe'],
            body: "{\"aliases\":{\"myapp_deflector-test.template\":{\"is_write_index\":true}}}").
       to_return(:status => 200, :body => "", :headers => {})
@@ -928,7 +944,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/logstash").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
         with(basic_auth: ['john', 'doe'],
              :body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
@@ -995,7 +1011,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/logstash").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
         with(basic_auth: ['john', 'doe'],
              :body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
@@ -1061,7 +1077,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
         with(basic_auth: ['john', 'doe'],
              :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
@@ -1126,7 +1142,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/_alias/logstash").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/#{alias_endpoint}/logstash").
         with(basic_auth: ['john', 'doe'],
              :body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
@@ -1193,7 +1209,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/logstash").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
         with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
         to_return(status: 200, body: "", headers: {})
       stub_request(:get, "https://logs.google.com:777/es//_xpack").
@@ -1257,7 +1273,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/logstash-test").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-test").
         with(basic_auth: ['john', 'doe'],
              :body => "{\"aliases\":{\"logstash-test\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
@@ -1441,7 +1457,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     # put the alias for the index
-    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
@@ -1450,7 +1466,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
   end
 
-    def test_custom_template_with_rollover_index_create_and_deflector_alias
+  def test_custom_template_with_rollover_index_create_and_deflector_alias
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_alias_template.json')
 
@@ -1492,7 +1508,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     # put the alias for the index
-    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
@@ -1544,7 +1560,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     # put the alias for the index
-    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs-#{timestr}").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
@@ -1615,7 +1631,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
              body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
         to_return(status: 200, body: "", headers: {})
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
         with(basic_auth: ['john', 'doe'],
              :body => "{\"aliases\":{\"mylogs\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
@@ -1686,7 +1702,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
              body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
         to_return(status: 200, body: "", headers: {})
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
         with(basic_auth: ['john', 'doe'],
              :body => "{\"aliases\":{\"mylogs\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
@@ -1756,7 +1772,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
              body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-*\"}").
         to_return(status: 200, body: "", headers: {})
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
         with(basic_auth: ['john', 'doe'],
              :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
@@ -1825,7 +1841,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
              body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-custom-test-*\"}").
         to_return(status: 200, body: "", headers: {})
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs-custom-test").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-custom-test").
         with(basic_auth: ['john', 'doe'],
              :body => "{\"aliases\":{\"mylogs-custom-test\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
@@ -1902,7 +1918,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe']).
         to_return(status: 200, body: "", headers: {})
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
         with(basic_auth: ['john', 'doe'],
              :body => "{\"aliases\":{\"mylogs\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
@@ -2034,7 +2050,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     # put the alias for the index
-    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp_deflector").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
@@ -2548,6 +2564,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
 
   def test_writes_to_default_index_with_compression
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+
     config = %[
       compression_level default_compression
     ]
@@ -3121,6 +3139,54 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
+  class LogStashDateformatPlaceholdersTest < self
+    def test_writes_to_logstash_index_with_specified_prefix_and_dateformat_placeholder_pattern_1
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'logstash_format' => true,
+                           'logstash_dateformat' => '${indexformat}',
+                           'logstash_prefix' => 'myprefix',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,time,indexformat', {
+                                                         'chunk_keys' => ['tag', 'time', 'indexformat'],
+                                                         'timekey' => 3600,
+                                                       }, [])
+                         ]
+                       ))
+      time = Time.parse Date.today.iso8601
+      logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
+      stub_elastic
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record.merge('indexformat' => '%Y.%m.%d'))
+      end
+      assert_equal(logstash_index, index_cmds.first['index']['_index'])
+    end
+
+    def test_writes_to_logstash_index_with_specified_prefix_and_dateformat_placeholder_pattern_2
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'logstash_format' => true,
+                           'logstash_dateformat' => '${indexformat}',
+                           'logstash_prefix' => 'myprefix',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,time,indexformat', {
+                                                         'chunk_keys' => ['tag', 'time', 'indexformat'],
+                                                         'timekey' => 3600,
+                                                       }, [])
+                         ]
+                       ))
+      time = Time.parse Date.today.iso8601
+      logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
+      stub_elastic
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record.merge('indexformat' => '%Y.%m'))
+      end
+      assert_equal(logstash_index, index_cmds.first['index']['_index'])
+    end
+  end
+
   class HostnamePlaceholders < self
     def test_writes_to_extracted_host
       driver.configure("host ${tag}\n")

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -118,6 +118,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'configure compression' do
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+
     config = %{
       compression_level best_compression
     }
@@ -127,6 +129,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'check compression strategy' do
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+
     config = %{
       compression_level best_speed
     }
@@ -136,6 +140,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'check content-encoding header with compression' do
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+
     config = %{
       compression_level best_compression
     }
@@ -145,6 +151,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'check compression option is passed to transport' do
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+
     config = %{
       compression_level best_compression
     }
@@ -341,6 +349,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def test_writes_to_default_index_with_compression
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+
     config = %[
       compression_level default_compression
     ]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.4
+  version: 4.0.5
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-19 00:00:00.000000000 Z
+date: 2020-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -148,6 +148,7 @@ files:
 - Rakefile
 - appveyor.yml
 - fluent-plugin-elasticsearch.gemspec
+- gemfiles/Gemfile.elasticsearch.v6
 - gemfiles/Gemfile.ilm
 - lib/fluent/log-ext.rb
 - lib/fluent/plugin/default-ilm-policy.json
@@ -170,6 +171,7 @@ files:
 - test/plugin/test_elasticsearch_tls.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_in_elasticsearch.rb
+- test/plugin/test_oj_serializer.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
 - test/plugin/test_template.json
@@ -206,6 +208,7 @@ test_files:
 - test/plugin/test_elasticsearch_tls.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_in_elasticsearch.rb
+- test/plugin/test_oj_serializer.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
 - test/plugin/test_template.json