fluent-plugin-opensearch 1.0.5 → 1.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0dc7a8e74a02c8940f19855c7377b7022e62ba0a9645e63eb2df366b0e085bb1
-  data.tar.gz: 482ed0c66522385c60a0765fbd0852e6158050ed4f803b7fed8a7045faf67beb
+  metadata.gz: ba58a770b6f7d4aa621091bee5315988649ef54d2bb4b4c23e980d9ee8598008
+  data.tar.gz: 5f6023c9456af019bf82141180a76b01bc322418456c191ceae72c7f794fe44f
 SHA512:
-  metadata.gz: ddab26f5bd880a43a29a345db493e7ceb39aafd5d6a583fd895975808041ee2856b125d579ffa64e8716f46e340e6a0e8d4079bc790b9a8b60f3ebcba1f53168
-  data.tar.gz: 4e2e566e012edcab487b66c6e5ca2cfe0e0e87d4470bbc79704d830ee561789a16e537967d549a1d0ea687cf59a7cf86781de1ae7dfdd15928934a2fc86ecae0
+  metadata.gz: 98474ac3e7329082aace9f0d671c0ecf60844fa0bab91c431be8c0dc8ddd0bcfeca60ef0612d8c7e79459a4da528f266ecce0a42eea5029d28fce7483d6e87ec
+  data.tar.gz: 5879925be1f7b7079eb7f7c44d50145aab2f1b76afb6f99d3345252d25f96ce7b8fcf3bd1a215252d6e7a1ab76cc727a02bc6b2aaa854e9f98e969c485e7df61

data/History.md

@@ -2,6 +2,15 @@
 
 ### [Unreleased]
 
+### 1.0.8
+ - Use faraday 1.x explicitly (#71)
+
+### 1.0.7
+ - Expire AWS credentials with a certain interval (#52)
+
+### 1.0.6
+ - out\_opensearch: Handle suppress\_type\_name operation correctly (#61)
+
 ### 1.0.5
  -  Use if clause to detect requirements for emitting error label events (#57)
  - opensearch_data_stream: Align lowercases for data_stream and its template names (#50)

data/README.md

@@ -1432,7 +1432,7 @@ Specify AWS OpenSearch Service endpoint.
 </endpoint>
 ```
 
-**NOTE:** This plugin will remove trailing slashes automatically. You don't need to pay attension to the trailing slash characters.
+**NOTE:** This plugin will remove trailing slashes automatically. You don't need to pay attention to the trailing slash characters.
 
 #### access_key_id
 
@@ -1564,6 +1564,19 @@ In this case, the endpoint configuration looks like:
 </endpoint>
 ```
 
+### Expiring AWS credentials
+
+If you want to expire AWS credentials in certain interval, you should specify `refresh_credentials_interval` parameter under `endpoint` section:
+
+```aconf
+<endpoint>
+  url https://CLUSTER_ENDPOINT_URL
+  region eu-west-1
+  # ...
+  refresh_credentials_interval 3h # default is 5h (five hours).
+</endpoint>
+```
+
 ## Troubleshooting
 
 See [Troubleshooting document](README.Troubleshooting.md)

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.0.5'
+  s.version       = '1.0.8'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'excon', '>= 0'
   s.add_runtime_dependency 'opensearch-ruby'
   s.add_runtime_dependency "aws-sdk-core", "~> 3"
+  s.add_runtime_dependency "faraday", "~> 1.10"
   s.add_runtime_dependency "faraday_middleware-aws-sigv4"
 
 

data/lib/fluent/plugin/out_opensearch.rb

@@ -194,6 +194,7 @@ module Fluent::Plugin
       config_param :assume_role_session_name, :string, :default => "fluentd"
       config_param :assume_role_web_identity_token_file, :string, :default => nil
       config_param :sts_credentials_region, :string, :default => nil
+      config_param :refresh_credentials_interval, :time, :default => "5h"
     end
 
     config_section :buffer do
@@ -336,6 +337,22 @@ module Fluent::Plugin
           }
         end
       end
+      # If AWS credentials is set, consider to expire credentials information forcibly before expired.
+      @credential_mutex = Mutex.new
+      if @endpoint
+        @_aws_credentials = aws_credentials(@endpoint)
+
+        if @endpoint.refresh_credentials_interval
+          timer_execute(:out_opensearch_expire_credentials, @endpoint.refresh_credentials_interval) do
+            log.debug('Recreate the AWS credentials')
+
+            @credential_mutex.synchronize do
+              @_os = nil
+              @_aws_credentials = aws_credentials(@endpoint)
+            end
+          end
+        end
+      end
 
       @serializer_class = nil
       begin
@@ -607,7 +624,7 @@ module Fluent::Plugin
                              :aws_sigv4,
                              service: 'es',
                              region: @endpoint.region,
-                             credentials: aws_credentials(@endpoint),
+                             credentials: @_aws_credentials,
                            )
 
                            f.adapter @http_backend, @backend_options
@@ -989,7 +1006,7 @@ module Fluent::Plugin
         end
       end
 
-      if @suppress_type_name
+      if @suppress_type_name || @last_seen_major_version >= 2
         target_type = nil
       else
         # OpenSearch only supports "_doc".
@@ -998,7 +1015,7 @@ module Fluent::Plugin
 
       meta.clear
       meta["_index".freeze] = target_index
-      meta["_type".freeze] = target_type
+      meta["_type".freeze] = target_type unless target_type.nil?
       meta["_alias".freeze] = target_index_alias
 
       if @pipeline

data/test/plugin/test_out_opensearch.rb

@@ -2320,8 +2320,9 @@ class OpenSearchOutputTest < Test::Unit::TestCase
   end
 
   data(
-    "OpenSearch default"=> {"os_version" => 1, "_type" => "_doc", "suppress_type" => false},
-    "Suppressed type" => {"os_version" => 1, "_type" => nil, "suppress_type" => true},
+    "OpenSearch default" => {"os_version" => 1, "_type" => "_doc", "suppress_type" => false},
+    "Suppressed type"    => {"os_version" => 1, "_type" => nil,    "suppress_type" => true},
+    "OpenSearch 2"       => {"os_version" => 2, "_type" => nil,    "suppress_type" => true},
   )
   def test_writes_to_speficied_type(data)
     driver('', data["os_version"]).configure("suppress_type_name #{data['suppress_type']}")
@@ -2330,7 +2331,12 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal(data['_type'], index_cmds.first['index']['_type'])
+    if data["suppress_type"] || data["os_version"] >= 2
+      assert_false(index_cmds.first['index'].has_key?("_type"))
+    else
+      assert_true(index_cmds.first['index'].has_key?("_type"))
+      assert_equal(data['_type'], index_cmds.first['index']['_type'])
+    end
   end
 
   def test_writes_to_speficied_host

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.8
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-25 00:00:00.000000000 Z
+date: 2022-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: faraday_middleware-aws-sigv4
   requirement: !ruby/object:Gem::Requirement