fluent-plugin-elasticsearch 4.0.10 → 4.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a12dc6596a56b7486d45129b64187021215433d2e7aa5428e96712fb63a345b5
-  data.tar.gz: 6fe5296f4a0afa23e863399f6d1cc9eeaa8f70c55e5cb979ca8db1bbf7a8988f
+  metadata.gz: c90e0b347a5c90ec23231df927384dadcc4b91b813a4e2c11df23539dfe05628
+  data.tar.gz: d6e296c8006870a5f85bd0b97294c69d797fc3f40b574ede7645fcd808ae6356
 SHA512:
-  metadata.gz: 4f5d200eb1dee56f9f4fc05efc46cdbf440e0e2df7152e527985b124ac91ff74b0d263bee36f69fddec9f5ccc3730032145d51a95cc22fc00817f909fd4c2a7a
-  data.tar.gz: 11b0453ea6cbcfaf46e71866a96daf4ceba8f3877dd23e7aa124ea214f056b1b7f56bd8082ae273883a7403a2f1752c64beb377a2a3564c1d30757a8a5f101fa
+  metadata.gz: 8466c4e3a13ef5c5ca1bda536a0baf7044fdaa0bc3ae1e1c7e6d0e6cf3dc7596d2b68e3f61ed2fb788f8ef2d8392939b8f63fd6aedc53e5818725bc6e0910aa6
+  data.tar.gz: b0572e50952ce49e44b280e94764d0d204d482e8e89cc2124e9a3025c1bd4cf7debe5625ad141337e5f81e6b6816286ee81f931f6b782f0a8cffcbfa0ed12dc3

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 4.0.11
+- Add custom and time placeholders combination testcase for ILM (#781)
+- Really support ILM on `logstash_format` enabled environment (#779)
+
 ### 4.0.10
 - filter_elasticsearch_genid: Use entire record as hash seed (#777)
 - Suppress type in meta with suppress_type_name parameter (#774)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.10'
+  s.version       = '4.0.11'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -761,9 +761,9 @@ EOC
         begin
           meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
           info = if @include_index_in_url
-                   RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze])
+                   RequestInfo.new(host, meta.delete("_index".freeze), meta.delete("_alias".freeze))
                  else
-                   RequestInfo.new(host, nil, meta["_index".freeze])
+                   RequestInfo.new(host, nil, meta.delete("_alias".freeze))
                  end
 
           if split_request?(bulk_message, info)
@@ -809,7 +809,7 @@ EOC
     end
 
     def process_message(tag, meta, header, time, record, extracted_values)
-      logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline, _ilm_policy_id = extracted_values
+      logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
 
       if @flatten_hashes
         record = flatten_record(record)
@@ -832,17 +832,19 @@ EOC
 
       target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
       if target_index_parent && target_index_parent[target_index_child_key]
-        target_index = target_index_parent.delete(target_index_child_key)
+        target_index_alias = target_index = target_index_parent.delete(target_index_child_key)
       elsif @logstash_format
         dt = dt.new_offset(0) if @utc_index
         target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
+        target_index_alias = "#{logstash_prefix}#{@logstash_prefix_separator}#{application_name}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
       else
-        target_index = index_name
+        target_index_alias = target_index = index_name
       end
 
       # Change target_index to lower-case since Elasticsearch doesn't
       # allow upper-case characters in index names.
       target_index = target_index.downcase
+      target_index_alias = target_index_alias.downcase
       if @include_tag_key
         record[@tag_key] = tag
       end
@@ -877,6 +879,7 @@ EOC
       meta.clear
       meta["_index".freeze] = target_index
       meta["_type".freeze] = target_type unless @last_seen_major_version >= 8
+      meta["_alias".freeze] = target_index_alias
 
       if @pipeline
         meta["pipeline".freeze] = pipeline

data/test/plugin/test_out_elasticsearch.rb

@@ -1053,6 +1053,80 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
     end
 
+    def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_template.json')
+
+      config = %{
+        host            logs.google.com
+        port            777
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        template_name   logstash
+        template_file   #{template_file}
+        index_date_pattern now/w{xxxx.ww}
+        enable_ilm      true
+        logstash_format true
+        application_name log
+      }
+
+      date_str = Time.now.strftime("%Y.%m.%d")
+      # connection start
+      stub_request(:head, "https://logs.google.com:777/es//").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-log-#{date_str}").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
+        with(basic_auth: ['john', 'doe'],
+             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
+        to_return(status: 200, body: "", headers: {})
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-log-#{date_str}").
+        with(basic_auth: ['john', 'doe'],
+             body: "{\"aliases\":{\"logstash-log-#{date_str}\":{\"is_write_index\":true}}}").
+        to_return(status: 200, body: "", headers: {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      driver(config)
+
+      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+      driver.run(default_tag: 'test') do
+        driver.feed(sample_record)
+      end
+      assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}", times: 1)
+
+      assert_requested(elastic_request)
+    end
+
     def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
       cwd = File.dirname(__FILE__)
       template_file = File.join(cwd, 'test_template.json')
@@ -1607,6 +1681,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
       assert_requested(elastic_request)
     end
+
+    def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_template.json')
+
+      config = Fluent::Config::Element.new(
+        'ROOT', '', {
+          '@type' => 'elasticsearch',
+          'host' => 'logs.google.com',
+          'port' => 777,
+          'scheme' => "https",
+          'path' => "/es/",
+          'user' => 'john',
+          'password' => 'doe',
+          'template_name' => 'logstash',
+          'template_file' => "#{template_file}",
+          'index_date_pattern' => 'now/w{xxxx.ww}',
+          'index_name' => "${taskDef}-%Y.%m",
+          'enable_ilm' => true,
+        }, [
+          Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
+                                        'chunk_keys' => ['tag', 'time', 'taskDef'],
+                                        'timekey' => 3600,
+                                      }, [])
+        ]
+      )
+
+      task_def_value = "task_definition"
+      date_str = Time.now.strftime("%Y.%m")
+      # connection start
+      stub_request(:head, "https://logs.google.com:777/es//").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
+        with(basic_auth: ['john', 'doe'],
+             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      driver(config)
+
+      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+      driver.run(default_tag: 'test') do
+        driver.feed(sample_record.merge("taskDef" => task_def_value))
+      end
+      assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
+
+      assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
+
+      assert_requested(elastic_request)
+    end
   end
 
   def test_custom_template_create
@@ -1863,11 +2014,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
     # check if alias exists
     timestr = Time.now.strftime("%Y.%m.%d")
-    stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
+    stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-myapp-#{timestr}").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     # put the alias for the index
-    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-myapp-#{timestr}").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.10
+  version: 4.0.11
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-06 00:00:00.000000000 Z
+date: 2020-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd