fluent-plugin-opensearch 1.1.0 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ecc83833c1b081c2c7a43eb518ec339c2fe2b1c3153ea424d02b046bfd26129c
-  data.tar.gz: '09443e78f21f9e3a064c0fdc0faea5d20d2b67ce85f8f187cef5322382a827e5'
+  metadata.gz: 1b8cc79e0b3618ac9af026312e4d73784ea6bc939e944f03bcae487a239a3106
+  data.tar.gz: 67a5c76a80dc1f00bb13febd1feac5ab029393934d8f2e60b74f6af9c96e03d4
 SHA512:
-  metadata.gz: fa530480b103bacbe4e00bc0bff3d3aca549acb126fc2251affa80267e24391bd929f9c9a903906c83807cdddc880a6a53b6470ec944fcaaecd73943bcb3560a
-  data.tar.gz: bc8a40ae17f566b1176edf1f163e8df740df72c19f99fce220dcc0a88cc1c857bfd5b1014f201f5261328eee234d3eb24a2421cc7b882e6c1b047e7e660fb4f3
+  metadata.gz: 2dadfc649ab1f9836031963ee85cdfc739029f71f35bdbd44b55f61c35351ffe1f84b64f118d6adf16ae866ebb9bc479fdcce3d71d2d0f83baba2f4fba013e80
+  data.tar.gz: e3a1a7df140ec59197d231b86a291dd1686d47ab2817da3983e3e9931b5e71e5994b138a6b2f10e1191a9f4deaba458c5cd221e541e66cbd534f533de85ec633

data/History.md

@@ -2,6 +2,13 @@
 
 ### [Unreleased]
 
+### 1.1.2
+ - Check OS cluster for data streams and templates for index template creation (#106)
+ - out\_opensearch\_data\_stream: Don't connect to opensearch on dry-run (#105)
+
+### 1.1.1
+ -  Pass a value of refresh\_credentials\_interval as duration\_seconds (#78)
+
 ### 1.1.0
  - Unpin `faraday` from v1, upgrade to v2.
    Note that if you can't migrate other plugins from `faraday` v1 yet, need to keep

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.1.0'
+  s.version       = '1.1.2'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_opensearch.rb

@@ -83,6 +83,7 @@ module Fluent::Plugin
     attr_reader :template_names
     attr_reader :ssl_version_options
     attr_reader :compressable_connection
+    attr_reader :duration_seconds
 
     helpers :event_emitter, :compat_parameters, :record_accessor, :timer
 
@@ -94,6 +95,7 @@ module Fluent::Plugin
     DEFAULT_RELOAD_AFTER = -1
     DEFAULT_TARGET_BULK_BYTES = -1
     DEFAULT_POLICY_ID = "logstash-policy"
+    DEFAULT_DURATION = "5h"
 
     config_param :host, :string,  :default => 'localhost'
     config_param :port, :integer, :default => 9200
@@ -195,7 +197,7 @@ module Fluent::Plugin
       config_param :assume_role_session_name, :string, :default => "fluentd"
       config_param :assume_role_web_identity_token_file, :string, :default => nil
       config_param :sts_credentials_region, :string, :default => nil
-      config_param :refresh_credentials_interval, :time, :default => "5h"
+      config_param :refresh_credentials_interval, :time, :default => DEFAULT_DURATION
       config_param :aws_service_name, :enum, list: [:es, :aoss], :default => :es
     end
 
@@ -211,6 +213,8 @@ module Fluent::Plugin
 
     def initialize
       super
+
+      @duration_seconds = Fluent::Config.time_value(DEFAULT_DURATION)
     end
 
     ######################################################################################################
@@ -238,13 +242,15 @@ module Fluent::Plugin
             credentials = Aws::AssumeRoleCredentials.new({
                             role_arn: conf[:assume_role_arn],
                             role_session_name: conf[:assume_role_session_name],
-                            region: sts_creds_region(conf)
+                            region: sts_creds_region(conf),
+                            duration_seconds: @duration_seconds
                           }).credentials
           else
             credentials = Aws::AssumeRoleWebIdentityCredentials.new({
                             role_arn: conf[:assume_role_arn],
                             web_identity_token_file: conf[:assume_role_web_identity_token_file],
-                            region: sts_creds_region(conf)
+                            region: sts_creds_region(conf),
+                            duration_seconds: @duration_seconds
                           }).credentials
           end
         end
@@ -345,7 +351,18 @@ module Fluent::Plugin
         @_aws_credentials = aws_credentials(@endpoint)
 
         if @endpoint.refresh_credentials_interval
-          timer_execute(:out_opensearch_expire_credentials, @endpoint.refresh_credentials_interval) do
+          @duration_seconds = Fluent::Config.time_value(@endpoint.refresh_credentials_interval)
+          # 60 * 60 * 12 = 12 hours
+          if @duration_seconds > 43200
+            raise Fluent::ConfigError, "Maximum duration is 12 hours."
+          end
+
+          # 60 * 15 = 15 minutes
+          if @duration_seconds < 900
+            raise Fluent::ConfigError, "Minimum duration is 15 minutes."
+          end
+
+          timer_execute(:out_opensearch_expire_credentials, @duration_seconds) do
             log.debug('Recreate the AWS credentials')
 
             @credential_mutex.synchronize do

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -69,23 +69,26 @@ module Fluent::Plugin
 
     def create_index_template(datastream_name, template_name, host = nil)
       # Create index template from file
-      if @template_file
-        template_installation_actual(template_name, @customize_template, @application_name, datastream_name, host)
-      else # Create default index template
-        return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
-        body = {
-          "index_patterns" => ["#{datastream_name}*"],
-          "data_stream" => {},
-        }
-
-        params = {
-          name: template_name,
-          body: body
-        }
-        retry_operate(@max_retry_putting_template,
-                      @fail_on_putting_template_retry_exceed,
-                      @catch_transport_exception_on_retry) do
-          client(host).indices.put_index_template(params)
+      if !dry_run?
+        if @template_file
+          return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
+          template_installation_actual(template_name, @customize_template, @application_name, datastream_name, host)
+        else # Create default index template
+          return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
+          body = {
+            "index_patterns" => ["#{datastream_name}*"],
+            "data_stream" => {},
+          }
+
+          params = {
+            name: template_name,
+            body: body
+          }
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
+            client(host).indices.put_index_template(params)
+          end
         end
       end
     end
@@ -159,13 +162,10 @@ module Fluent::Plugin
                end
         data_stream_name = extract_placeholders(@data_stream_name, chunk).downcase
         data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk).downcase
-        unless @data_stream_names.include?(data_stream_name)
-          begin
-            create_index_template(data_stream_name, data_stream_template_name, host)
-            @data_stream_names << data_stream_name
-          rescue => e
-            raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
-          end
+        begin
+          create_index_template(data_stream_name, data_stream_template_name, host)
+        rescue => e
+          raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
         end
       end
 

data/test/plugin/test_out_opensearch.rb

@@ -300,6 +300,7 @@ class OpenSearchOutputTest < Test::Unit::TestCase
                                       'region' => "local",
                                       'access_key_id' => 'YOUR_AWESOME_KEY',
                                       'secret_access_key' => 'YOUR_AWESOME_SECRET',
+                                      'refresh_credentials_interval' => '10h'
                                     }, []),
         Fluent::Config::Element.new('buffer', 'tag', {}, [])
 
@@ -316,6 +317,8 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     assert_nil instance.endpoint.assume_role_web_identity_token_file
     assert_nil instance.endpoint.sts_credentials_region
     assert_equal :es, instance.endpoint.aws_service_name
+    assert_equal 36000, instance.endpoint.refresh_credentials_interval
+    assert_equal 36000, instance.duration_seconds
   end
 
   data("OpenSearch Service" => [:es, 'es'],

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.2
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-11 00:00:00.000000000 Z
+date: 2023-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -270,7 +270,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Opensearch output plugin for Fluent event collector