fluent-plugin-elasticsearch 5.4.2 → 5.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d654e4c24122a1f08b7440556ff43eca647e9aae360c4ebc43f4e29d7a8abfec
-  data.tar.gz: 961f8dab627d2628e854e4b6b64a30fb3719c8387dfd73a67aa6da8f059eb447
+  metadata.gz: 542a7706becd1b24f8d3e6ad8957015dba5f721b3a93047ec4bea80313df8200
+  data.tar.gz: 8104645d96578abf0f4357a22c83a7f9cbc548e7bbc35ef5eeb8726deaada6f8
 SHA512:
-  metadata.gz: 78d3e7ab001e326c47a192798da97b9d44956db46869b807589818469b8c74cfc36bc936c67f2fdffed3c2f2967d4c9ede1737c8c6d774362f524861eee74186
-  data.tar.gz: 85e1144f3b227d28772cf785932a44bdf6d73307b7cc0401098acd97083b9638089d5a1e5ad2343cc9a7f64f4d93e274db99419fe07fdfe5b785176d667e6592
+  metadata.gz: 93a2a932a766170b197f3ded762c89912f743cec3ddcef3e7719d8ca57b504f6f7b51bc17bac02ef5073fd1fade3366140a7d775351c8b86b1f05693a76be122
+  data.tar.gz: 35b82c172909a1f47dc74507c7ea9df451576b9cb1c8dcba85fdb094b3568b2187aafa1aecbd3367ef2ab0a347a30f90d8b663780e3261bdefc5dcb2b24f0133

data/History.md

@@ -2,7 +2,10 @@
 
 ### [Unreleased]
 
-### 5.4.1
+### 5.4.3
+- add data_stream_template\_use\_index\_patterns\_wildcard in elasticsearch\_data\_stream (#1040)
+
+### 5.4.2
 - in\_elasticsearch: Avoid to use deprecated endpoint for clear\_scroll (#1039)
 
 ### 5.4.1

data/README.md

@@ -1561,6 +1561,50 @@ Default value is `false`.
 
 **NOTE:** This parameter requests to install elasticsearch-xpack gem.
 
+### data_stream_template_use_index_patterns_wildcard
+
+Specify whether index patterns should include a wildcard (*) when creating an index template. This is particularly useful to prevent errors in scenarios where index templates are generated automatically, and multiple services with distinct suffixes are in use.
+
+Default value is `true`.
+
+Consider the following JSON error response when index patterns clash due to wildcard usage:
+```json
+{
+  "error": {
+    "root_cause": [
+      {
+        "type": "illegal_argument_exception",
+        "reason": "index template [eks-kube-apiserver] has index patterns [eks-kube-apiserver*] matching patterns from existing templates [eks-kube-apiserver-audit] with patterns (eks-kube-apiserver-audit => [eks-kube-apiserver-audit*]) that have the same priority [0], multiple index templates may not match during index creation, please use a different priority"
+      }
+    ],
+    "type": "illegal_argument_exception",
+    "reason": "index template [eks-kube-apiserver] has index patterns [eks-kube-apiserver*] matching patterns from existing templates [eks-kube-apiserver-audit] with patterns (eks-kube-apiserver-audit => [eks-kube-apiserver-audit*]) that have the same priority [0], multiple index templates may not match during index creation, please use a different priority"
+  },
+  "status": 400
+}
+```
+
+#### Usage Examples
+
+When `data_stream_template_use_index_patterns_wildcard` is set to `true` (default):
+
+```
+data_stream_name: foo
+data_stream_template_use_index_patterns_wildcard: true
+```
+
+In this case, the resulting index patterns will be: `["foo*"]`
+
+When `data_stream_template_use_index_patterns_wildcard` is set to `false`:
+
+```
+data_stream_name: foo
+data_stream_template_use_index_patterns_wildcard: false
+```
+
+The resulting index patterns will be: `["foo"]`
+
+
 ## Troubleshooting
 
 See [Troubleshooting document](README.Troubleshooting.md)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.4.2'
+  s.version       = '5.4.3'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -13,6 +13,7 @@ module Fluent::Plugin
     config_param :data_stream_template_name, :string, :default => nil
     config_param :data_stream_ilm_policy, :string, :default => nil
     config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
+    config_param :data_stream_template_use_index_patterns_wildcard, :bool, :default => true
 
     # Elasticsearch 7.9 or later always support new style of index template.
     config_set_default :use_legacy_template, false
@@ -112,8 +113,9 @@ module Fluent::Plugin
 
     def create_index_template(datastream_name, template_name, ilm_name, host = nil)
       return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
+      wildcard = @data_stream_template_use_index_patterns_wildcard ? '*' : ''
       body = {
-        "index_patterns" => ["#{datastream_name}*"],
+        "index_patterns" => ["#{datastream_name}#{wildcard}"],
         "data_stream" => {},
         "template" => {
           "settings" => {

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -34,6 +34,10 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     '_ilm'.freeze
   end
 
+  def index_template_endpoint
+    '_index_template'.freeze
+  end
+
   def driver(conf='', es_version=elasticsearch_version.to_i, client_version=elasticsearch_version)
     # For request stub to detect compatibility.
     @es_version ||= es_version
@@ -124,7 +128,6 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound], :headers => {'x-elastic-product' => 'Elasticsearch'})
   end
 
-
   def push_bulk_request(req_body)
     # bulk data must be pair of OP and records
     # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
@@ -486,7 +489,8 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
         '@type' => ELASTIC_DATA_STREAM_TYPE,
         'data_stream_name' => 'foo',
         'data_stream_ilm_name' => "foo_ilm_policy",
-        'data_stream_template_name' => "foo_tpl"
+        'data_stream_template_name' => "foo_tpl",
+        'data_stream_template_use_index_patterns_wildcard' => false
       })
     assert_equal "foo", driver(conf).instance.data_stream_name
   end
@@ -556,6 +560,21 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal true, driver(config).instance.compression
   end
 
+  def test_configure_wildcard_usage
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name                                        foo
+      data_stream_template_name                               foo_tpl
+      data_stream_ilm_name                                    foo_ilm_policy
+      data_stream_template_use_index_patterns_wildcard        false
+    }
+
+    stub_default
+
+    assert_equal false, driver(config).instance.data_stream_template_use_index_patterns_wildcard
+  end
+
   def test_check_compression_strategy
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
@@ -722,6 +741,72 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal "foo_policy", driver(conf).instance.data_stream_ilm_name
   end
 
+  def test_template_index_patterns_with_wildcard
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_existent_ilm?
+
+    stub_nonexistent_data_stream?
+    stub_data_stream
+
+    stub_nonexistent_template?("foo_template")
+    stub_request(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template").with do |req|
+      # bulk data must be pair of OP and records
+      # {"create": {}}\nhttp://localhost:9200/_index_template//foo_template
+      # {"@timestamp": ...}
+      push_bulk_request(req.body)
+    end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
+
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => ELASTIC_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_ilm_name' => 'foo_ilm_policy',
+      'data_stream_template_use_index_patterns_wildcard' => false
+    })
+
+    assert_nothing_raised {
+      driver(conf)
+    }
+
+    assert_requested(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template",
+                     body: '{"index_patterns":["foo"],"data_stream":{},"template":{"settings":{"index.lifecycle.name":"foo_ilm_policy"}}}',
+                     times: 1)
+  end
+
+  def test_template_index_patterns_without_wildcard
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_existent_ilm?
+
+    stub_nonexistent_data_stream?
+    stub_data_stream
+
+    stub_nonexistent_template?("foo_template")
+    stub_request(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template").with do |req|
+      # bulk data must be pair of OP and records
+      # {"create": {}}\nhttp://localhost:9200/_index_template//foo_template
+      # {"@timestamp": ...}
+      push_bulk_request(req.body)
+    end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
+
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => ELASTIC_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_ilm_name' => 'foo_ilm_policy',
+      'data_stream_template_use_index_patterns_wildcard' => true
+    })
+
+    assert_nothing_raised {
+      driver(conf)
+    }
+
+    assert_requested(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template",
+                     body: '{"index_patterns":["foo*"],"data_stream":{},"template":{"settings":{"index.lifecycle.name":"foo_ilm_policy"}}}',
+                     times: 1)
+  end
+
   def test_placeholder
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.4.2
+  version: 5.4.3
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-05 00:00:00.000000000 Z
+date: 2024-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd