fluent-plugin-opensearch 1.0.4 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d2bf385e2b57f561be89d6ccb74da8a95870668368643321c9217a83683d3ca
-  data.tar.gz: a5e622af13e62991eac0845ef4e0d33e12252f18c5073d335b78b8dc7055c2f1
+  metadata.gz: c44bf1e420cb64daf8b5a2f21f38c47e692ed31feb0bd400189e737c19e819df
+  data.tar.gz: 896882755f7394472a8a4874a7a48786c50aee729b283e7be32818edcacc3e0c
 SHA512:
-  metadata.gz: 88ad2347a77fdd8a95851f53514def3396e424634a1e906c7700b875000881e5cd6825d8441cc765912e838dc9b2068702949d6722343fac0d476b9993fb3c35
-  data.tar.gz: b0f852ea5f3b42778ebfcad0d32a657a554077806e3e500dc9eb4bb7625cc95188aded8437d6ba7ca43eeafdecfa4dd40f01220f2956d78b9eb9b476d2e6ece9
+  metadata.gz: a0718cd172d5549040faad8698a09eeeb9f5aadb00e19ab994d406143fa8789f20ce18e5b7526d9697822036c39f99aa96bdeeaace50b0b412f209d4057a3f5b
+  data.tar.gz: 59f7c8c735f4df3f69d4bc618df0653f00db191bd7c85f1a498509486f3fd7040073763436b0833279e24bdc52cf57b0a61322e896257a0e6ba6f19af9bc49e1

data/History.md

@@ -2,6 +2,16 @@
 
 ### [Unreleased]
 
+### 1.0.7
+ - Expire AWS credentials with a certain interval (#52)
+
+### 1.0.6
+ - out\_opensearch: Handle suppress\_type\_name operation correctly (#61)
+
+### 1.0.5
+ -  Use if clause to detect requirements for emitting error label events (#57)
+ - opensearch_data_stream: Align lowercases for data_stream and its template names (#50)
+
 ### 1.0.4
  - Automatically create data streams (#44)
 

data/README.md

@@ -1564,6 +1564,19 @@ In this case, the endpoint configuration looks like:
 </endpoint>
 ```
 
+### Expiring AWS credentials
+
+If you want to expire AWS credentials in certain interval, you should specify `refresh_credentials_interval` parameter under `endpoint` section:
+
+```aconf
+<endpoint>
+  url https://CLUSTER_ENDPOINT_URL
+  region eu-west-1
+  # ...
+  refresh_credentials_interval 3h # default is 5h (five hours).
+</endpoint>
+```
+
 ## Troubleshooting
 
 See [Troubleshooting document](README.Troubleshooting.md)

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.0.4'
+  s.version       = '1.0.7'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/opensearch_error_handler.rb

@@ -65,11 +65,8 @@ class Fluent::Plugin::OpenSearchErrorHandler
     end
   end
 
-  def emit_error_label_event(&block)
-    # If `emit_error_label_event` is specified as false, error event emittions are not occurred.
-    if @plugin.emit_error_label_event
-      block.call
-    end
+  def emit_error_label_event?
+    !!@plugin.emit_error_label_event
   end
 
   def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
@@ -138,14 +135,14 @@ class Fluent::Plugin::OpenSearchErrorHandler
             reason += " [reason]: \'#{item[write_operation]['error']['reason']}\'"
           end
         end
-        emit_error_label_event do
+        if emit_error_label_event?
           @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("400 - Rejected by OpenSearch#{reason}"))
         end
       else
         if item[write_operation]['error'].is_a?(String)
           reason = item[write_operation]['error']
           stats[:errors_block_resp] += 1
-          emit_error_label_event do
+          if emit_error_label_event?
             @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{reason}"))
           end
           next
@@ -156,7 +153,7 @@ class Fluent::Plugin::OpenSearchErrorHandler
             raise OpenSearchRequestAbortError, "Rejected OpenSearch due to #{type}"
           end
           if unrecoverable_record_error?(type)
-            emit_error_label_event do
+            if emit_error_label_event?
               @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{type}: #{reason}"))
             end
             next
@@ -167,7 +164,7 @@ class Fluent::Plugin::OpenSearchErrorHandler
           # When we don't have a type field, something changed in the API
           # expected return values.
           stats[:errors_bad_resp] += 1
-          emit_error_label_event do
+          if emit_error_label_event?
             @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - No error type provided in the response"))
           end
           next

data/lib/fluent/plugin/out_opensearch.rb

@@ -194,6 +194,7 @@ module Fluent::Plugin
       config_param :assume_role_session_name, :string, :default => "fluentd"
       config_param :assume_role_web_identity_token_file, :string, :default => nil
       config_param :sts_credentials_region, :string, :default => nil
+      config_param :refresh_credentials_interval, :time, :default => "5h"
     end
 
     config_section :buffer do
@@ -336,6 +337,22 @@ module Fluent::Plugin
           }
         end
       end
+      # If AWS credentials is set, consider to expire credentials information forcibly before expired.
+      @credential_mutex = Mutex.new
+      if @endpoint
+        @_aws_credentials = aws_credentials(@endpoint)
+
+        if @endpoint.refresh_credentials_interval
+          timer_execute(:out_opensearch_expire_credentials, @endpoint.refresh_credentials_interval) do
+            log.debug('Recreate the AWS credentials')
+
+            @credential_mutex.synchronize do
+              @_os = nil
+              @_aws_credentials = aws_credentials(@endpoint)
+            end
+          end
+        end
+      end
 
       @serializer_class = nil
       begin
@@ -465,11 +482,8 @@ module Fluent::Plugin
       placeholder_validities.include?(true)
     end
 
-    def emit_error_label_event(&block)
-      # If `emit_error_label_event` is specified as false, error event emittions are not occurred.
-      if @emit_error_label_event
-        block.call
-      end
+    def emit_error_label_event?
+      !!@emit_error_label_event
     end
 
     def compression
@@ -588,7 +602,7 @@ module Fluent::Plugin
     def parse_time(value, event_time, tag)
       @time_parser.call(value)
     rescue => e
-      emit_error_label_event do
+      if emit_error_label_event?
         router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
       end
       return Time.at(event_time).to_datetime
@@ -610,7 +624,7 @@ module Fluent::Plugin
                              :aws_sigv4,
                              service: 'es',
                              region: @endpoint.region,
-                             credentials: aws_credentials(@endpoint),
+                             credentials: @_aws_credentials,
                            )
 
                            f.adapter @http_backend, @backend_options
@@ -882,7 +896,7 @@ module Fluent::Plugin
             end
           end
         rescue => e
-          emit_error_label_event do
+          if emit_error_label_event?
             router.emit_error_event(tag, time, record, e)
           end
         end
@@ -992,7 +1006,7 @@ module Fluent::Plugin
         end
       end
 
-      if @suppress_type_name
+      if @suppress_type_name || @last_seen_major_version >= 2
         target_type = nil
       else
         # OpenSearch only supports "_doc".
@@ -1001,7 +1015,7 @@ module Fluent::Plugin
 
       meta.clear
       meta["_index".freeze] = target_index
-      meta["_type".freeze] = target_type
+      meta["_type".freeze] = target_type unless target_type.nil?
       meta["_alias".freeze] = target_index_alias
 
       if @pipeline

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -157,8 +157,8 @@ module Fluent::Plugin
                else
                  extract_placeholders(@host, chunk)
                end
-        data_stream_name = extract_placeholders(@data_stream_name, chunk)
-        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
+        data_stream_name = extract_placeholders(@data_stream_name, chunk).downcase
+        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk).downcase
         unless @data_stream_names.include?(data_stream_name)
           begin
             create_index_template(data_stream_name, data_stream_template_name, host)

data/test/plugin/test_out_opensearch.rb

@@ -2320,8 +2320,9 @@ class OpenSearchOutputTest < Test::Unit::TestCase
   end
 
   data(
-    "OpenSearch default"=> {"os_version" => 1, "_type" => "_doc", "suppress_type" => false},
-    "Suppressed type" => {"os_version" => 1, "_type" => nil, "suppress_type" => true},
+    "OpenSearch default" => {"os_version" => 1, "_type" => "_doc", "suppress_type" => false},
+    "Suppressed type"    => {"os_version" => 1, "_type" => nil,    "suppress_type" => true},
+    "OpenSearch 2"       => {"os_version" => 2, "_type" => nil,    "suppress_type" => true},
   )
   def test_writes_to_speficied_type(data)
     driver('', data["os_version"]).configure("suppress_type_name #{data['suppress_type']}")
@@ -2330,7 +2331,12 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal(data['_type'], index_cmds.first['index']['_type'])
+    if data["suppress_type"] || data["os_version"] >= 2
+      assert_false(index_cmds.first['index'].has_key?("_type"))
+    else
+      assert_true(index_cmds.first['index'].has_key?("_type"))
+      assert_equal(data['_type'], index_cmds.first['index']['_type'])
+    end
   end
 
   def test_writes_to_speficied_host

data/test/plugin/test_out_opensearch_data_stream.rb

@@ -439,6 +439,23 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal 1, @bulk_records
   end
 
+  def test_placeholder_with_capital_letters
+    dsname = "foo_test.capital_letters"
+    tplname = "foo_tpl_test.capital_letters"
+    stub_default(dsname, tplname)
+    stub_bulk_feed(dsname, tplname)
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo_${tag}',
+        'data_stream_template_name' => "foo_tpl_${tag}"
+      })
+    driver(conf).run(default_tag: 'TEST.CAPITAL_LETTERS') do
+      driver.feed(sample_record)
+    end
+    assert_equal 1, @bulk_records
+  end
+
   def test_placeholder_params_unset
     dsname = "foo_test"
     tplname = "foo_test_template"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.7
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-12 00:00:00.000000000 Z
+date: 2022-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd