fluent-plugin-opensearch 1.0.3 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 217feaf328809b259c6eb716e968b02417df7b10075a9a1e278c6e337d8a660b
-  data.tar.gz: 351327bd18d5dd3aa7b6a360ebba4a4f574a86b65ed470cffdf15b38e5d16022
+  metadata.gz: 988e07d44cf92e9f0a09dfa57191ac4acdfd3a3fb1cd9598f12cb3e196ea88ac
+  data.tar.gz: 786b826f8fdb04d2c16804c42efaa4c40042deb43f71bcb6bed31de13742b413
 SHA512:
-  metadata.gz: c374adb923d9533e50c1c11355b304ba04ca92112dee36dcacd886d53be0ec910481559c7fdb0f6cdb3b61c7c949b5b917c3540d4bbd2789a9d568015cb52cbb
-  data.tar.gz: 11259ac1ee5e4dfe0de5b18336fd2167b40b8c10c381a283068557da1d206e06c03d5d76ef46411a8803b00c333370f47c21fbfdf4331e528cb05a145803a9b4
+  metadata.gz: 9a6c39ce68b3d8616d490cd04006c85397ccf0fc4bb768f5f8182dad3f63abf5d29f721928ee7f074b361ba44d529e7080f99814300949ea5464ce4279ee1ba8
+  data.tar.gz: 8eb1f4a48740d1d85dfeb0d25de3877c2e4dcf4f228d0bc3f4907c952aad17327ee4f157972338dad21ce38a0b5b44571553b918aa096191444772c18ef6f2d9

data/History.md

@@ -2,6 +2,16 @@
 
 ### [Unreleased]
 
+### 1.0.6
+ - out\_opensearch: Handle suppress\_type\_name operation correctly (#61)
+
+### 1.0.5
+ -  Use if clause to detect requirements for emitting error label events (#57)
+ - opensearch_data_stream: Align lowercases for data_stream and its template names (#50)
+
+### 1.0.4
+ - Automatically create data streams (#44)
+
 ### 1.0.3
  - Configurable unrecoverable record types (#40)
  - Handle exceptions on retrieving AWS credentials (#39)

data/README.md

@@ -1047,11 +1047,11 @@ $ fluentd -r $sniffer [AND YOUR OTHER OPTIONS]
 The default selector used by the `OpenSearch::Transport` class works well when Fluentd should behave round robin and random selector cases. This doesn't work well when Fluentd should behave fallbacking from exhausted ES cluster to normal ES cluster.
 The parameter `selector_class_name` gives you the ability to provide your own Selector class to implement whatever selection nodes logic you require.
 
-The below configuration is using plugin built-in `ElasticseatchFallbackSelector`:
+The below configuration is using plugin built-in `OpenSearchFallbackSelector`:
 
 ```
 hosts exhausted-host:9201,normal-host:9200
-selector_class_name "Fluent::Plugin::ElasticseatchFallbackSelector"
+selector_class_name "Fluent::Plugin::OpenSeartchFallbackSelector"
 ```
 
 #### Tips

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.0.3'
+  s.version       = '1.0.6'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/opensearch_error_handler.rb

@@ -65,11 +65,8 @@ class Fluent::Plugin::OpenSearchErrorHandler
     end
   end
 
-  def emit_error_label_event(&block)
-    # If `emit_error_label_event` is specified as false, error event emittions are not occurred.
-    if @plugin.emit_error_label_event
-      block.call
-    end
+  def emit_error_label_event?
+    !!@plugin.emit_error_label_event
   end
 
   def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
@@ -138,14 +135,14 @@ class Fluent::Plugin::OpenSearchErrorHandler
             reason += " [reason]: \'#{item[write_operation]['error']['reason']}\'"
           end
         end
-        emit_error_label_event do
+        if emit_error_label_event?
           @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("400 - Rejected by OpenSearch#{reason}"))
         end
       else
         if item[write_operation]['error'].is_a?(String)
           reason = item[write_operation]['error']
           stats[:errors_block_resp] += 1
-          emit_error_label_event do
+          if emit_error_label_event?
             @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{reason}"))
           end
           next
@@ -156,7 +153,7 @@ class Fluent::Plugin::OpenSearchErrorHandler
             raise OpenSearchRequestAbortError, "Rejected OpenSearch due to #{type}"
           end
           if unrecoverable_record_error?(type)
-            emit_error_label_event do
+            if emit_error_label_event?
               @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{type}: #{reason}"))
             end
             next
@@ -167,7 +164,7 @@ class Fluent::Plugin::OpenSearchErrorHandler
           # When we don't have a type field, something changed in the API
           # expected return values.
           stats[:errors_bad_resp] += 1
-          emit_error_label_event do
+          if emit_error_label_event?
             @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - No error type provided in the response"))
           end
           next

data/lib/fluent/plugin/opensearch_index_template.rb

@@ -62,10 +62,10 @@ module Fluent::OpenSearchIndexTemplate
     client.transport.transport.host_unreachable_exceptions
   end
 
-  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
+  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_transport_exceptions = true)
     return unless block_given?
     retries = 0
-    transport_errors = OpenSearch::Transport::Transport::Errors.constants.map{ |c| OpenSearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
+    transport_errors = OpenSearch::Transport::Transport::Errors.constants.map{ |c| OpenSearch::Transport::Transport::Errors.const_get c } if catch_transport_exceptions
     begin
       yield
     rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e

data/lib/fluent/plugin/out_opensearch.rb

@@ -465,11 +465,8 @@ module Fluent::Plugin
       placeholder_validities.include?(true)
     end
 
-    def emit_error_label_event(&block)
-      # If `emit_error_label_event` is specified as false, error event emittions are not occurred.
-      if @emit_error_label_event
-        block.call
-      end
+    def emit_error_label_event?
+      !!@emit_error_label_event
     end
 
     def compression
@@ -588,7 +585,7 @@ module Fluent::Plugin
     def parse_time(value, event_time, tag)
       @time_parser.call(value)
     rescue => e
-      emit_error_label_event do
+      if emit_error_label_event?
         router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
       end
       return Time.at(event_time).to_datetime
@@ -882,7 +879,7 @@ module Fluent::Plugin
             end
           end
         rescue => e
-          emit_error_label_event do
+          if emit_error_label_event?
             router.emit_error_event(tag, time, record, e)
           end
         end
@@ -992,7 +989,7 @@ module Fluent::Plugin
         end
       end
 
-      if @suppress_type_name
+      if @suppress_type_name || @last_seen_major_version >= 2
         target_type = nil
       else
         # OpenSearch only supports "_doc".
@@ -1001,7 +998,7 @@ module Fluent::Plugin
 
       meta.clear
       meta["_index".freeze] = target_index
-      meta["_type".freeze] = target_type
+      meta["_type".freeze] = target_type unless target_type.nil?
       meta["_alias".freeze] = target_index_alias
 
       if @pipeline

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -36,7 +36,6 @@ module Fluent::Plugin
                         @fail_on_putting_template_retry_exceed,
                         @catch_transport_exception_on_retry) do
             create_index_template(@data_stream_name, @data_stream_template_name)
-            create_data_stream(@data_stream_name)
           end
         rescue => e
           raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
@@ -105,19 +104,6 @@ module Fluent::Plugin
       end
     end
 
-    def create_data_stream(datastream_name, host = nil)
-      return if data_stream_exist?(datastream_name, host)
-      params = {
-        name: datastream_name
-      }
-      retry_operate(@max_retry_putting_template,
-                    @fail_on_putting_template_retry_exceed,
-                    @catch_transport_exception_on_retry) do
-        # TODO: Use X-Pack equivalent performing DataStream operation method on the following line
-        client(host).perform_request('PUT', "/_data_stream/#{datastream_name}", {}, params)
-      end
-    end
-
     def template_exists?(name, host = nil)
       if @use_legacy_template
         client(host).indices.get_template(:name => name)
@@ -171,12 +157,11 @@ module Fluent::Plugin
                else
                  extract_placeholders(@host, chunk)
                end
-        data_stream_name = extract_placeholders(@data_stream_name, chunk)
-        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
+        data_stream_name = extract_placeholders(@data_stream_name, chunk).downcase
+        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk).downcase
         unless @data_stream_names.include?(data_stream_name)
           begin
             create_index_template(data_stream_name, data_stream_template_name, host)
-            create_data_stream(data_stream_name, host)
             @data_stream_names << data_stream_name
           rescue => e
             raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"

data/test/plugin/test_out_opensearch.rb

@@ -2320,8 +2320,9 @@ class OpenSearchOutputTest < Test::Unit::TestCase
   end
 
   data(
-    "OpenSearch default"=> {"os_version" => 1, "_type" => "_doc", "suppress_type" => false},
-    "Suppressed type" => {"os_version" => 1, "_type" => nil, "suppress_type" => true},
+    "OpenSearch default" => {"os_version" => 1, "_type" => "_doc", "suppress_type" => false},
+    "Suppressed type"    => {"os_version" => 1, "_type" => nil,    "suppress_type" => true},
+    "OpenSearch 2"       => {"os_version" => 2, "_type" => nil,    "suppress_type" => true},
   )
   def test_writes_to_speficied_type(data)
     driver('', data["os_version"]).configure("suppress_type_name #{data['suppress_type']}")
@@ -2330,7 +2331,12 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal(data['_type'], index_cmds.first['index']['_type'])
+    if data["suppress_type"] || data["os_version"] >= 2
+      assert_false(index_cmds.first['index'].has_key?("_type"))
+    else
+      assert_true(index_cmds.first['index'].has_key?("_type"))
+      assert_equal(data['_type'], index_cmds.first['index']['_type'])
+    end
   end
 
   def test_writes_to_speficied_host

data/test/plugin/test_out_opensearch_data_stream.rb

@@ -439,6 +439,23 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal 1, @bulk_records
   end
 
+  def test_placeholder_with_capital_letters
+    dsname = "foo_test.capital_letters"
+    tplname = "foo_tpl_test.capital_letters"
+    stub_default(dsname, tplname)
+    stub_bulk_feed(dsname, tplname)
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo_${tag}',
+        'data_stream_template_name' => "foo_tpl_${tag}"
+      })
+    driver(conf).run(default_tag: 'TEST.CAPITAL_LETTERS') do
+      driver.feed(sample_record)
+    end
+    assert_equal 1, @bulk_records
+  end
+
   def test_placeholder_params_unset
     dsname = "foo_test"
     tplname = "foo_test_template"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.6
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-25 00:00:00.000000000 Z
+date: 2022-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -242,7 +242,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.30
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Opensearch output plugin for Fluent event collector