fluent-plugin-opensearch 1.0.2 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6382f3ad5be4b3c2c5aaa425568e4421c761bcb8cc5de65369123d01830a8888
-  data.tar.gz: 3e38aceba420423bf15065a0203ed6d696fa862d86d365a090d72d8da290a92a
+  metadata.gz: 0dc7a8e74a02c8940f19855c7377b7022e62ba0a9645e63eb2df366b0e085bb1
+  data.tar.gz: 482ed0c66522385c60a0765fbd0852e6158050ed4f803b7fed8a7045faf67beb
 SHA512:
-  metadata.gz: 0bccfd9d20535f856b4dd115eb9126efc2cf2cfbe02d52fcb1f2e14db43747ee90de71f87873924540ea973f1d077d4618b87beb053e09e4d1c52d7b1b33bcc6
-  data.tar.gz: 3d9a7ce81fbb624f4af5e42dcfef10e4059073523a5b410af67575cffb11bf7e243f11d66cc6bf87851ea5e5dd0f2f7d76100923d53cf386df82ede3beb0c3ad
+  metadata.gz: ddab26f5bd880a43a29a345db493e7ceb39aafd5d6a583fd895975808041ee2856b125d579ffa64e8716f46e340e6a0e8d4079bc790b9a8b60f3ebcba1f53168
+  data.tar.gz: 4e2e566e012edcab487b66c6e5ca2cfe0e0e87d4470bbc79704d830ee561789a16e537967d549a1d0ea687cf59a7cf86781de1ae7dfdd15928934a2fc86ecae0

data/History.md

@@ -2,6 +2,20 @@
 
 ### [Unreleased]
 
+### 1.0.5
+ -  Use if clause to detect requirements for emitting error label events (#57)
+ - opensearch_data_stream: Align lowercases for data_stream and its template names (#50)
+
+### 1.0.4
+ - Automatically create data streams (#44)
+
+### 1.0.3
+ - Configurable unrecoverable record types (#40)
+ - Handle exceptions on retrieving AWS credentials (#39)
+ - Suppress emit error label events (#38)
+ - Provide suppress_type_name parameter (#29)
+ - Honor @time_key (data streams) (#28)
+
 ### 1.0.2
  - Honor @hosts parameter for Data Streams (#21)
  - Use template_file for Data Streams (#20)

data/README.OpenSearchInput.md

@@ -17,6 +17,7 @@
   + [reload_on_failure](#reload_on_failure)
   + [resurrect_after](#resurrect_after)
   + [with_transporter_log](#with_transporter_log)
+  + [emit_error_label_event](#emit-error-label-event)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [sniffer_class_name](#sniffer-class-name)
   + [custom_headers](#custom_headers)
@@ -190,6 +191,18 @@ We recommend to set this true if you start to debug this plugin.
 with_transporter_log true
 ```
 
+### emit_error_label_event
+
+Default `emit_error_label_event` value is `true`.
+
+Emitting error label events is default behavior.
+
+When using the followin configuration, OpenSearch plugin will cut error events on error handler:
+
+```aconf
+emit_error_label_event false
+```
+
 ### Client/host certificate options
 
 Need to verify OpenSearch's certificate?  You can use the following parameter to specify a CA instead of using an environment variable.

data/README.md

@@ -76,6 +76,8 @@ Send your logs to OpenSearch (and search them with OpenSearch Dashboards maybe?)
   + [reload_after](#reload-after)
   + [validate_client_version](#validate-client-version)
   + [unrecoverable_error_types](#unrecoverable-error-types)
+  + [unrecoverable_record_types](#unrecoverable-record-types)
+  + [emit_error_label_event](#emit-error-label-event)
   + [verify os version at startup](#verify_os_version_at_startup)
   + [default_opensearch_version](#default_opensearch_version)
   + [custom_headers](#custom_headers)
@@ -356,6 +358,15 @@ utc_index true
 
 By default, the records inserted into index `logstash-YYMMDD` with UTC (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
 
+
+### suppress_type_name
+
+If OpenSearch cluster complains types removal warnings, this can be suppressed with:
+
+```
+suppress_type_name true
+```
+
 ### target_index_key
 
 Tell this plugin to find the index name to write to in the record under this key in preference to other mechanisms. Key can be specified as path to nested record using dot ('.') as a separator.
@@ -1036,11 +1047,11 @@ $ fluentd -r $sniffer [AND YOUR OTHER OPTIONS]
 The default selector used by the `OpenSearch::Transport` class works well when Fluentd should behave round robin and random selector cases. This doesn't work well when Fluentd should behave fallbacking from exhausted ES cluster to normal ES cluster.
 The parameter `selector_class_name` gives you the ability to provide your own Selector class to implement whatever selection nodes logic you require.
 
-The below configuration is using plugin built-in `ElasticseatchFallbackSelector`:
+The below configuration is using plugin built-in `OpenSearchFallbackSelector`:
 
 ```
 hosts exhausted-host:9201,normal-host:9200
-selector_class_name "Fluent::Plugin::ElasticseatchFallbackSelector"
+selector_class_name "Fluent::Plugin::OpenSeartchFallbackSelector"
 ```
 
 #### Tips
@@ -1097,6 +1108,33 @@ Then, remove `rejected_execution_exception` from `unrecoverable_error_types` par
 unrecoverable_error_types ["out_of_memory_error"]
 ```
 
+
+### Unrecoverable Record Types
+
+Default `unrecoverable_record_types` parameter is set up loosely.
+Because `json_parse_exception` is caused by invalid JSON record.
+Another possible unrecoverable record error should be included within this paramater.
+
+If you want to handle `security_exception` as _unrecoverable exceptions_, please consider to change `unrecoverable_record_types` parameter from default value:
+
+```
+unrecoverable_record_types ["json_parse_exception", "security_exception"]
+```
+
+If this error type is included in OpenSearch response, an invalid record should be sent in `@ERROR` data pipeline.
+
+### emit_error_label_event
+
+Default `emit_error_label_event` value is `true`.
+
+Emitting error label events is default behavior.
+
+When using the followin configuration, OpenSearch plugin will cut error events on error handler:
+
+```aconf
+emit_error_label_event false
+```
+
 ### verify_os_version_at_startup
 
 Because OpenSearch plugin will ought to change behavior each of OpenSearch major versions.

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.0.2'
+  s.version       = '1.0.5'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/in_opensearch.rb

@@ -73,6 +73,7 @@ module Fluent::Plugin
     config_param :ca_file, :string, :default => nil
     config_param :ssl_version, :enum, list: [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2], :default => :TLSv1_2
     config_param :with_transporter_log, :bool, :default => false
+    config_param :emit_error_label_event, :bool, :default => true
     config_param :sniffer_class_name, :string, :default => nil
     config_param :custom_headers, :hash, :default => {}
     config_param :docinfo_fields, :array, :default => ['_index', '_type', '_id']
@@ -180,6 +181,13 @@ module Fluent::Plugin
       }
     end
 
+    def emit_error_label_event(&block)
+      # If `emit_error_label_event` is specified as false, error event emittions are not occurred.
+      if emit_error_label_event
+        block.call
+      end
+    end
+
     def start
       super
 
@@ -224,7 +232,9 @@ module Fluent::Plugin
     def parse_time(value, event_time, tag)
       @timestamp_parser.call(value)
     rescue => e
-      router.emit_error_event(@timestamp_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @timestamp_key_format, 'value' => value}, e)
+      emit_error_label_event do
+        router.emit_error_event(@timestamp_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @timestamp_key_format, 'value' => value}, e)
+      end
       return Time.at(event_time).to_time
     end
 

data/lib/fluent/plugin/opensearch_error_handler.rb

@@ -49,8 +49,12 @@ class Fluent::Plugin::OpenSearchErrorHandler
     unrecoverable_error_types.include?(type)
   end
 
+  def unrecoverable_record_types
+    @plugin.unrecoverable_record_types
+  end
+
   def unrecoverable_record_error?(type)
-    ['json_parse_exception'].include?(type)
+    unrecoverable_record_types.include?(type)
   end
 
   def log_os_400_reason(&block)
@@ -61,6 +65,10 @@ class Fluent::Plugin::OpenSearchErrorHandler
     end
   end
 
+  def emit_error_label_event?
+    !!@plugin.emit_error_label_event
+  end
+
   def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
     items = response['items']
     if items.nil? || !items.is_a?(Array)
@@ -127,12 +135,16 @@ class Fluent::Plugin::OpenSearchErrorHandler
             reason += " [reason]: \'#{item[write_operation]['error']['reason']}\'"
           end
         end
-        @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("400 - Rejected by OpenSearch#{reason}"))
+        if emit_error_label_event?
+          @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("400 - Rejected by OpenSearch#{reason}"))
+        end
       else
         if item[write_operation]['error'].is_a?(String)
           reason = item[write_operation]['error']
           stats[:errors_block_resp] += 1
-          @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{reason}"))
+          if emit_error_label_event?
+            @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{reason}"))
+          end
           next
         elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
@@ -141,7 +153,9 @@ class Fluent::Plugin::OpenSearchErrorHandler
             raise OpenSearchRequestAbortError, "Rejected OpenSearch due to #{type}"
           end
           if unrecoverable_record_error?(type)
-            @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{type}: #{reason}"))
+            if emit_error_label_event?
+              @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{type}: #{reason}"))
+            end
             next
           else
             retry_stream.add(time, rawrecord) unless unrecoverable_record_error?(type)
@@ -150,7 +164,9 @@ class Fluent::Plugin::OpenSearchErrorHandler
           # When we don't have a type field, something changed in the API
           # expected return values.
           stats[:errors_bad_resp] += 1
-          @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - No error type provided in the response"))
+          if emit_error_label_event?
+            @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - No error type provided in the response"))
+          end
           next
         end
         stats[type] += 1

data/lib/fluent/plugin/opensearch_index_template.rb

@@ -62,10 +62,10 @@ module Fluent::OpenSearchIndexTemplate
     client.transport.transport.host_unreachable_exceptions
   end
 
-  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
+  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_transport_exceptions = true)
     return unless block_given?
     retries = 0
-    transport_errors = OpenSearch::Transport::Transport::Errors.constants.map{ |c| OpenSearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
+    transport_errors = OpenSearch::Transport::Transport::Errors.constants.map{ |c| OpenSearch::Transport::Transport::Errors.const_get c } if catch_transport_exceptions
     begin
       yield
     rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e

data/lib/fluent/plugin/out_opensearch.rb

@@ -110,6 +110,7 @@ module Fluent::Plugin
     config_param :logstash_prefix_separator, :string, :default => '-'
     config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
     config_param :utc_index, :bool, :default => true
+    config_param :suppress_type_name, :bool, :default => false
     config_param :index_name, :string, :default => "fluentd"
     config_param :id_key, :string, :default => nil
     config_param :write_operation, :string, :default => "index"
@@ -160,6 +161,8 @@ module Fluent::Plugin
     config_param :validate_client_version, :bool, :default => false
     config_param :prefer_oj_serializer, :bool, :default => false
     config_param :unrecoverable_error_types, :array, :default => ["out_of_memory_error", "rejected_execution_exception"]
+    config_param :unrecoverable_record_types, :array, :default => ["json_parse_exception"]
+    config_param :emit_error_label_event, :bool, :default => true
     config_param :verify_os_version_at_startup, :bool, :default => true
     config_param :default_opensearch_version, :integer, :default => DEFAULT_OPENSEARCH_VERSION
     config_param :log_os_400_reason, :bool, :default => false
@@ -219,8 +222,8 @@ module Fluent::Plugin
         if conf[:assume_role_arn].nil?
           aws_container_credentials_relative_uri = conf[:ecs_container_credentials_relative_uri] || ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
           if aws_container_credentials_relative_uri.nil?
-            credentials = Aws::SharedCredentials.new({retries: 2}).credentials
-            credentials ||= Aws::InstanceProfileCredentials.new.credentials
+            credentials = Aws::SharedCredentials.new({retries: 2}).credentials rescue nil
+            credentials ||= Aws::InstanceProfileCredentials.new.credentials rescue nil
             credentials ||= Aws::ECSCredentials.new.credentials
           else
             credentials = Aws::ECSCredentials.new({
@@ -462,6 +465,10 @@ module Fluent::Plugin
       placeholder_validities.include?(true)
     end
 
+    def emit_error_label_event?
+      !!@emit_error_label_event
+    end
+
     def compression
       !(@compression_level == :no_compression)
     end
@@ -578,7 +585,9 @@ module Fluent::Plugin
     def parse_time(value, event_time, tag)
       @time_parser.call(value)
     rescue => e
-      router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
+      if emit_error_label_event?
+        router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
+      end
       return Time.at(event_time).to_datetime
     end
 
@@ -870,7 +879,9 @@ module Fluent::Plugin
             end
           end
         rescue => e
-          router.emit_error_event(tag, time, record, e)
+          if emit_error_label_event?
+            router.emit_error_event(tag, time, record, e)
+          end
         end
       end
 
@@ -978,8 +989,12 @@ module Fluent::Plugin
         end
       end
 
-      # OpenSearch only supports "_doc".
-      target_type = DEFAULT_TYPE_NAME
+      if @suppress_type_name
+        target_type = nil
+      else
+        # OpenSearch only supports "_doc".
+        target_type = DEFAULT_TYPE_NAME
+      end
 
       meta.clear
       meta["_index".freeze] = target_index

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -36,7 +36,6 @@ module Fluent::Plugin
                         @fail_on_putting_template_retry_exceed,
                         @catch_transport_exception_on_retry) do
             create_index_template(@data_stream_name, @data_stream_template_name)
-            create_data_stream(@data_stream_name)
           end
         rescue => e
           raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
@@ -105,19 +104,6 @@ module Fluent::Plugin
       end
     end
 
-    def create_data_stream(datastream_name, host = nil)
-      return if data_stream_exist?(datastream_name, host)
-      params = {
-        name: datastream_name
-      }
-      retry_operate(@max_retry_putting_template,
-                    @fail_on_putting_template_retry_exceed,
-                    @catch_transport_exception_on_retry) do
-        # TODO: Use X-Pack equivalent performing DataStream operation method on the following line
-        client(host).perform_request('PUT', "/_data_stream/#{datastream_name}", {}, params)
-      end
-    end
-
     def template_exists?(name, host = nil)
       if @use_legacy_template
         client(host).indices.get_template(:name => name)
@@ -171,12 +157,11 @@ module Fluent::Plugin
                else
                  extract_placeholders(@host, chunk)
                end
-        data_stream_name = extract_placeholders(@data_stream_name, chunk)
-        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
+        data_stream_name = extract_placeholders(@data_stream_name, chunk).downcase
+        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk).downcase
         unless @data_stream_names.include?(data_stream_name)
           begin
             create_index_template(data_stream_name, data_stream_template_name, host)
-            create_data_stream(data_stream_name, host)
             @data_stream_names << data_stream_name
           rescue => e
             raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
@@ -191,12 +176,22 @@ module Fluent::Plugin
       tag = chunk.metadata.tag
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
-
         begin
-          record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
+          if record.has_key?(TIMESTAMP_FIELD)
+            rts = record[TIMESTAMP_FIELD]
+            dt = parse_time(rts, time, tag)
+          elsif record.has_key?(@time_key)
+            rts = record[@time_key]
+            dt = parse_time(rts, time, tag)
+          else
+            dt = Time.at(time).to_datetime
+          end
+          record.merge!({"@timestamp" => dt.iso8601(@time_precision)})
           bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
         rescue => e
-          router.emit_error_event(tag, time, record, e)
+          emit_error_label_event do
+            router.emit_error_event(tag, time, record, e)
+          end
         end
       end
 

data/test/plugin/test_opensearch_error_handler.rb

@@ -35,14 +35,18 @@ class TestOpenSearchErrorHandler < Test::Unit::TestCase
     attr_reader :log
     attr_reader :error_events
     attr_accessor :unrecoverable_error_types
+    attr_accessor :unrecoverable_record_types
     attr_accessor :log_os_400_reason
     attr_accessor :write_operation
+    attr_accessor :emit_error_label_event
     def initialize(log, log_os_400_reason = false)
       @log = log
       @write_operation = 'index'
       @error_events = []
       @unrecoverable_error_types = ["out_of_memory_error", "rejected_execution_exception"]
+      @unrecoverable_record_types = ["json_parse_exception"]
       @log_os_400_reason = log_os_400_reason
+      @emit_error_label_event = true
     end
 
     def router
@@ -135,6 +139,44 @@ class TestOpenSearchErrorHandler < Test::Unit::TestCase
     end
   end
 
+  class TEST400ResponseReasonWithoutErrorLog < self
+    def setup
+      Fluent::Test.setup
+      @log_device = Fluent::Test::DummyLogDevice.new
+      dl_opts = {:log_level => ServerEngine::DaemonLogger::DEBUG}
+      logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
+      @log = Fluent::Log.new(logger)
+      @plugin = TestPlugin.new(@log)
+      @handler = Fluent::Plugin::OpenSearchErrorHandler.new(@plugin)
+      @plugin.emit_error_label_event = false
+    end
+
+    def test_400_responses_reason_log
+      records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
+      response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "status" : 400,
+            "error" : {
+              "type"  : "mapper_parsing_exception",
+              "reason" : "failed to parse"
+            }
+          }
+        }
+      ]
+     }))
+      chunk = MockChunk.new(records)
+      dummy_extracted_values = []
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      assert_equal(0, @plugin.error_events.size)
+      assert_true(@plugin.error_events.empty?)
+    end
+  end
+
   class TEST400ResponseReasonNoDebug < self
     def setup
       Fluent::Test.setup
@@ -177,6 +219,45 @@ class TestOpenSearchErrorHandler < Test::Unit::TestCase
     end
   end
 
+  class TEST400ResponseReasonNoDebugAndNoErrorLog < self
+    def setup
+      Fluent::Test.setup
+      @log_device = Fluent::Test::DummyLogDevice.new
+      dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
+      logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
+      @log = Fluent::Log.new(logger)
+      @plugin = TestPlugin.new(@log)
+      @handler = Fluent::Plugin::OpenSearchErrorHandler.new(@plugin)
+      @plugin.log_os_400_reason = true
+      @plugin.emit_error_label_event = false
+    end
+
+    def test_400_responses_reason_log
+      records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
+      response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "status" : 400,
+            "error" : {
+              "type"  : "mapper_parsing_exception",
+              "reason" : "failed to parse"
+            }
+          }
+        }
+      ]
+     }))
+      chunk = MockChunk.new(records)
+      dummy_extracted_values = []
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      assert_equal(0, @plugin.error_events.size)
+      assert_true(@plugin.error_events.empty?)
+    end
+  end
+
   def test_nil_items_responses
     records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
     response = parse_response(%({

data/test/plugin/test_out_opensearch.rb

@@ -2320,23 +2320,11 @@ class OpenSearchOutputTest < Test::Unit::TestCase
   end
 
   data(
-    "OpenSearch default"=> {"os_version" => 1, "_type" => "_doc"},
+    "OpenSearch default"=> {"os_version" => 1, "_type" => "_doc", "suppress_type" => false},
+    "Suppressed type" => {"os_version" => 1, "_type" => nil, "suppress_type" => true},
   )
   def test_writes_to_speficied_type(data)
-    driver('', data["os_version"]).configure("")
-    stub_opensearch
-    stub_opensearch_info
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
-    end
-    assert_equal(data['_type'], index_cmds.first['index']['_type'])
-  end
-
-  data(
-    "OpenSearch 1"=> {"os_version" => 1, "_type" => "_doc"},
-  )
-  def test_writes_to_speficied_type_with_placeholders(data)
-    driver('', data["os_version"]).configure("")
+    driver('', data["os_version"]).configure("suppress_type_name #{data['suppress_type']}")
     stub_opensearch
     stub_opensearch_info
     driver.run(default_tag: 'test') do

data/test/plugin/test_out_opensearch_data_stream.rb

@@ -9,7 +9,7 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
   include FlexMock::TestCase
   include Fluent::Test::Helpers
 
-  attr_accessor :bulk_records
+  attr_accessor :bulk_records, :index_cmds
 
   OPENSEARCH_DATA_STREAM_TYPE = "opensearch_data_stream"
 
@@ -96,12 +96,14 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
       # {"create": {}}\nhttp://localhost:9200/_data_stream/foo_bar
       # {"@timestamp": ...}
       @bulk_records += req.body.split("\n").size / 2
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end
     stub_request(:post, "http://#{url}#{template_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_data_stream/foo_bar
       # {"@timestamp": ...}
       @bulk_records += req.body.split("\n").size / 2
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end
   end
 
@@ -437,6 +439,23 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal 1, @bulk_records
   end
 
+  def test_placeholder_with_capital_letters
+    dsname = "foo_test.capital_letters"
+    tplname = "foo_tpl_test.capital_letters"
+    stub_default(dsname, tplname)
+    stub_bulk_feed(dsname, tplname)
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo_${tag}',
+        'data_stream_template_name' => "foo_tpl_${tag}"
+      })
+    driver(conf).run(default_tag: 'TEST.CAPITAL_LETTERS') do
+      driver.feed(sample_record)
+    end
+    assert_equal 1, @bulk_records
+  end
+
   def test_placeholder_params_unset
     dsname = "foo_test"
     tplname = "foo_test_template"
@@ -591,4 +610,72 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
 
     assert_equal(4, connection_resets)
   end
+
+  def test_uses_custom_time_key
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl',
+      'time_key' => 'vtm'
+    })
+
+    ts = DateTime.new(2021,2,3).iso8601(9)
+    record = {
+      'vtm' => ts,
+      'message' => 'Sample Record'
+    }
+
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(ts, index_cmds[1]['@timestamp'])
+  end
+
+  def test_uses_custom_time_key_with_format
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl',
+      'time_key' => 'vtm',
+      'time_key_format' => '%Y-%m-%d %H:%M:%S.%N%z'
+    })
+    ts = "2021-02-03 13:14:01.673+02:00"
+    record = {
+      'vtm' => ts,
+      'message' => 'Sample Record'
+    }
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(DateTime.parse(ts).iso8601(9), index_cmds[1]['@timestamp'])
+  end
+
+  def test_record_no_timestamp
+    stub_default
+    stub_bulk_feed
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl'
+    })
+    record = {
+      'message' => 'Sample Record'
+    }
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.5
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-17 00:00:00.000000000 Z
+date: 2022-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -242,7 +242,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.30
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Opensearch output plugin for Fluent event collector