fluent-plugin-elasticsearch 4.0.3 → 4.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 248fd3a633d1ee967b26d0d62e20a22f8ea7b67b16bfa1222001f8a3578ab94a
-  data.tar.gz: 0143d71c8043323f61ff94117ae973ea0f5889f4dcb3ae38156f06dcb1c0fd5d
+  metadata.gz: c4253b332821c59318a743dccc063cfe0bf5b6f5302727d0bf974280b41336ea
+  data.tar.gz: 8731c0ba69bb200b1c397fc87ad0237c16ca8f5a825d37d16c0844664afd6292
 SHA512:
-  metadata.gz: ab8e9c8ba00a928044fbabc61cda336ee3d61b6b38f34c6d1361610016d1c9e0ecd0c70817edda0bbb58e5b0861c2e401a50b85492d8e1a051034796d48ad2ee
-  data.tar.gz: bce6f61984f42957ca44e02e176cb7b7f11ff2959f774f38d167b978332f0e737a8edb4005c7f82a2abd717fb0aff898a068f5cfd25737c99006f7c245726224
+  metadata.gz: 41eff72fc9b79075b3e3c2812c43fc34d4e789ca005bbe0d722e6d75ef9d01706841bba7214495a1d1bcdc3eeb5d86008c002e4aa4686508b22f1caafce731a3
+  data.tar.gz: 3087f4fa5c6c008092366652ee388690bd012c6528ab01eb699fe352760a584500c83ea903d22b40693bf3d141bf7ed220f491cd64be3be08fa9a7ca2308f246

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 4.0.4
+- Provide clearing caches timer (#719)
+
 ### 4.0.3
 -  Use http.scheme settings for persisent scheme setup (#713)
 

data/README.md

@@ -92,6 +92,7 @@ Current maintainers: @cosmo0920
   + [ilm_policy_id](#ilm_policy_id)
   + [ilm_policy](#ilm_policy)
   + [ilm_policy_overwrite](#ilm_policy_overwrite)
+  + [truncate_caches_interval](#truncate_caches_interval)
 * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
@@ -1183,6 +1184,14 @@ Default value is `false`.
 
 **NOTE:** This parameter requests to install elasticsearch-xpack gem.
 
+## truncate_caches_interval
+
+Specify truncating caches interval.
+
+If it is set, timer for clearing `alias_indexes` and `template_names` caches will be launched and executed.
+
+Default value is `nil`.
+
 ## Configuration - Elasticsearch Input
 
 See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.3'
+  s.version       = '4.0.4'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -141,16 +141,17 @@ module Fluent::ElasticsearchIndexTemplate
         client.indices.put_alias(:index => index_name_temp, :name => deflector_alias_name,
                                  :body => body)
         log.info("The alias '#{deflector_alias_name}' is created for the index '#{index_name_temp}'")
-        if enable_ilm
-          if ilm_policy.empty?
-            setup_ilm(enable_ilm, ilm_policy_id)
-          else
-            setup_ilm(enable_ilm, ilm_policy_id, ilm_policy, ilm_policy_overwrite)
-          end
-        end
       else
         log.debug("The alias '#{deflector_alias_name}' is already present")
       end
+      # Create ILM policy if rollover indices exist.
+      if enable_ilm
+        if ilm_policy.empty?
+          setup_ilm(enable_ilm, ilm_policy_id)
+        else
+          setup_ilm(enable_ilm, ilm_policy_id, ilm_policy, ilm_policy_overwrite)
+        end
+      end
     else
       log.debug("No index and alias creation action performed because rollover_index or enable_ilm is set to: '#{rollover_index}', '#{enable_ilm}'")
     end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -56,7 +56,7 @@ module Fluent::Plugin
     attr_reader :template_names
     attr_reader :ssl_version_options
 
-    helpers :event_emitter, :compat_parameters, :record_accessor
+    helpers :event_emitter, :compat_parameters, :record_accessor, :timer
 
     Fluent::Plugin.register_output('elasticsearch', self)
 
@@ -158,6 +158,7 @@ EOC
     config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
     config_param :ilm_policy, :hash, :default => {}
     config_param :ilm_policy_overwrite, :bool, :default => false
+    config_param :truncate_caches_interval, :time, :default => nil
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -241,6 +242,18 @@ EOC
         end
       end
 
+      @truncate_mutex = Mutex.new
+      if @truncate_caches_interval
+        timer_execute(:out_elasticsearch_truncate_caches, @truncate_caches_interval) do
+          log.info('Clean up the indices and template names cache')
+
+          @truncate_mutex.synchronize {
+            @alias_indexes.clear
+            @template_names.clear
+          }
+        end
+      end
+
       @serializer_class = nil
       begin
         require 'oj'

data/test/plugin/test_out_elasticsearch.rb

@@ -810,6 +810,70 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(elastic_request)
   end
 
+  def test_template_create_with_rollover_index_and_template_related_placeholders_with_truncating_caches
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+    config = %{
+      host               logs.google.com
+      port               777
+      scheme             https
+      path               /es/
+      user               john
+      password           doe
+      template_name      logstash-${tag}
+      template_file      #{template_file}
+      rollover_index     true
+      index_date_pattern ""
+      index_name         fluentd-${tag}
+      deflector_alias    myapp_deflector-${tag}
+      truncate_caches_interval 2s
+    }
+
+    # connection start
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
+    to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash-test.template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 404, :body => "", :headers => {})
+    # create template
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash-test.template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if alias exists
+    stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector-test.template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 404, :body => "", :headers => {})
+    # put the alias for the index
+    stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E").
+      with(basic_auth: ['john', 'doe']).
+      to_return(status: 200, body: "", headers: {})
+    stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E/_alias/myapp_deflector-test.template").
+      with(basic_auth: ['john', 'doe'],
+           body: "{\"aliases\":{\"myapp_deflector-test.template\":{\"is_write_index\":true}}}").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver(config)
+
+    elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+    driver.run(default_tag: 'test.template', shutdown: false) do
+      driver.feed(sample_record)
+    end
+    assert_equal('fluentd-test.template', index_cmds.first['index']['_index'])
+
+    assert_equal ["myapp_deflector-test.template"], driver.instance.alias_indexes
+    assert_equal ["logstash-test.template"], driver.instance.template_names
+
+    assert_requested(elastic_request)
+
+    sleep 0.1 until driver.instance.alias_indexes.empty?
+    sleep 0.1 until driver.instance.template_names.empty?
+
+    assert_equal [], driver.instance.alias_indexes
+    assert_equal [], driver.instance.template_names
+  end
+
   class TemplateIndexLifecycleManagementTest < self
     def setup
       begin

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.3
+  version: 4.0.4
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-10 00:00:00.000000000 Z
+date: 2020-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd