fluent-plugin-opensearch 1.1.2 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b8cc79e0b3618ac9af026312e4d73784ea6bc939e944f03bcae487a239a3106
-  data.tar.gz: 67a5c76a80dc1f00bb13febd1feac5ab029393934d8f2e60b74f6af9c96e03d4
+  metadata.gz: 578e5c63061d5f1cb3f3729dee4ea630564dc60d82f6b047c8e4b23830fdf16a
+  data.tar.gz: a502d46eb5a3a59c5c4f65f9009623ff32fde1de4cfda7e181437a8720bdbda1
 SHA512:
-  metadata.gz: 2dadfc649ab1f9836031963ee85cdfc739029f71f35bdbd44b55f61c35351ffe1f84b64f118d6adf16ae866ebb9bc479fdcce3d71d2d0f83baba2f4fba013e80
-  data.tar.gz: e3a1a7df140ec59197d231b86a291dd1686d47ab2817da3983e3e9931b5e71e5994b138a6b2f10e1191a9f4deaba458c5cd221e541e66cbd534f533de85ec633
+  metadata.gz: 77b3373041396440cf8debaea8fd2ab95b94df950cbfe6105f65d45cce7092a6f89bd019b38d4022051c651541d3d21918625ea1b9939f58108101eae90a027f
+  data.tar.gz: 2e746cd797f5d24f08dbb386ea42b9bff506831c56f9f4b92dcf99dde321b9b613c7473cbad39bd1f6d338ae6dd094a87f838141306111ab49b5a76886346918

data/History.md

@@ -2,6 +2,13 @@
 
 ### [Unreleased]
 
+### 1.1.4
+ - test: remove minitest to correct misjudge of the framework by flexmock (#114)
+ - Add logic to write method of out_opensearch_data_stream (#109)
+
+### 1.1.3
+ - Revert the behavior of passing duration second (#108)
+
 ### 1.1.2
  - Check OS cluster for data streams and templates for index template creation (#106)
  - out\_opensearch\_data\_stream: Don't connect to opensearch on dry-run (#105)

data/README.md

@@ -1606,7 +1606,7 @@ There are usually a few feature requests, tagged [Easy](https://github.com/fluen
 
 Pull Requests are welcomed.
 
-Becore send a pull request or report an issue, please read [the contribution guideline](CONTRIBUTING.md).
+Before sending a pull request or reporting an issue, please read [the contribution guideline](CONTRIBUTING.md).
 
 ## Running tests
 

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.1.2'
+  s.version       = '1.1.4'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_opensearch.rb

@@ -83,7 +83,6 @@ module Fluent::Plugin
     attr_reader :template_names
     attr_reader :ssl_version_options
     attr_reader :compressable_connection
-    attr_reader :duration_seconds
 
     helpers :event_emitter, :compat_parameters, :record_accessor, :timer
 
@@ -95,7 +94,6 @@ module Fluent::Plugin
     DEFAULT_RELOAD_AFTER = -1
     DEFAULT_TARGET_BULK_BYTES = -1
     DEFAULT_POLICY_ID = "logstash-policy"
-    DEFAULT_DURATION = "5h"
 
     config_param :host, :string,  :default => 'localhost'
     config_param :port, :integer, :default => 9200
@@ -197,7 +195,7 @@ module Fluent::Plugin
       config_param :assume_role_session_name, :string, :default => "fluentd"
       config_param :assume_role_web_identity_token_file, :string, :default => nil
       config_param :sts_credentials_region, :string, :default => nil
-      config_param :refresh_credentials_interval, :time, :default => DEFAULT_DURATION
+      config_param :refresh_credentials_interval, :time, :default => "5h"
       config_param :aws_service_name, :enum, list: [:es, :aoss], :default => :es
     end
 
@@ -213,8 +211,6 @@ module Fluent::Plugin
 
     def initialize
       super
-
-      @duration_seconds = Fluent::Config.time_value(DEFAULT_DURATION)
     end
 
     ######################################################################################################
@@ -242,15 +238,13 @@ module Fluent::Plugin
             credentials = Aws::AssumeRoleCredentials.new({
                             role_arn: conf[:assume_role_arn],
                             role_session_name: conf[:assume_role_session_name],
-                            region: sts_creds_region(conf),
-                            duration_seconds: @duration_seconds
+                            region: sts_creds_region(conf)
                           }).credentials
           else
             credentials = Aws::AssumeRoleWebIdentityCredentials.new({
                             role_arn: conf[:assume_role_arn],
                             web_identity_token_file: conf[:assume_role_web_identity_token_file],
-                            region: sts_creds_region(conf),
-                            duration_seconds: @duration_seconds
+                            region: sts_creds_region(conf)
                           }).credentials
           end
         end
@@ -351,18 +345,7 @@ module Fluent::Plugin
         @_aws_credentials = aws_credentials(@endpoint)
 
         if @endpoint.refresh_credentials_interval
-          @duration_seconds = Fluent::Config.time_value(@endpoint.refresh_credentials_interval)
-          # 60 * 60 * 12 = 12 hours
-          if @duration_seconds > 43200
-            raise Fluent::ConfigError, "Maximum duration is 12 hours."
-          end
-
-          # 60 * 15 = 15 minutes
-          if @duration_seconds < 900
-            raise Fluent::ConfigError, "Minimum duration is 15 minutes."
-          end
-
-          timer_execute(:out_opensearch_expire_credentials, @duration_seconds) do
+          timer_execute(:out_opensearch_expire_credentials, @endpoint.refresh_credentials_interval) do
             log.debug('Recreate the AWS credentials')
 
             @credential_mutex.synchronize do

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -187,6 +187,12 @@ module Fluent::Plugin
             dt = Time.at(time).to_datetime
           end
           record.merge!({"@timestamp" => dt.iso8601(@time_precision)})
+          if @include_tag_key
+            record[@tag_key] = tag
+          end
+          if @remove_keys
+            @remove_keys.each { |key| record.delete(key) }
+          end
           bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
         rescue => e
           emit_error_label_event do

data/test/helper.rb

@@ -55,7 +55,6 @@ end
 
 require 'test/unit'
 require 'fluent/test'
-require 'minitest/pride'
 
 require 'webmock/test_unit'
 WebMock.disable_net_connect!

data/test/plugin/test_out_opensearch.rb

@@ -300,7 +300,6 @@ class OpenSearchOutputTest < Test::Unit::TestCase
                                       'region' => "local",
                                       'access_key_id' => 'YOUR_AWESOME_KEY',
                                       'secret_access_key' => 'YOUR_AWESOME_SECRET',
-                                      'refresh_credentials_interval' => '10h'
                                     }, []),
         Fluent::Config::Element.new('buffer', 'tag', {}, [])
 
@@ -317,8 +316,6 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     assert_nil instance.endpoint.assume_role_web_identity_token_file
     assert_nil instance.endpoint.sts_credentials_region
     assert_equal :es, instance.endpoint.aws_service_name
-    assert_equal 36000, instance.endpoint.refresh_credentials_interval
-    assert_equal 36000, instance.duration_seconds
   end
 
   data("OpenSearch Service" => [:es, 'es'],

data/test/plugin/test_out_opensearch_data_stream.rb

@@ -678,4 +678,69 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
     assert(index_cmds[1].has_key? '@timestamp')
   end
 
+  def test_record_with_include_tag_key
+    stub_default
+    stub_bulk_feed
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl',
+      'include_tag_key' => true,
+      'tag_key' => 'test_tag'
+    })
+    record = {
+      'message' => 'Sample Record'
+    }
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(index_cmds[1].has_key?('test_tag'))
+  end
+
+  def test_record_without_include_tag_key
+    stub_default
+    stub_bulk_feed
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl',
+      'include_tag_key' => false
+    })
+    record = {
+      'message' => 'Sample Record'
+    }
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(!index_cmds[1].has_key?('test'))
+  end
+
+  def test_record_with_remove_keys
+    stub_default
+    stub_bulk_feed
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl',
+      'remove_keys' => 'remove_me'
+    })
+    record = {
+      'message' => 'Sample Record',
+      'remove_me' => 'foo'
+    }
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(!index_cmds[1].has_key?('remove_me'))
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.4
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-28 00:00:00.000000000 Z
+date: 2023-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd