fluent-plugin-opensearch 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6382f3ad5be4b3c2c5aaa425568e4421c761bcb8cc5de65369123d01830a8888
-  data.tar.gz: 3e38aceba420423bf15065a0203ed6d696fa862d86d365a090d72d8da290a92a
+  metadata.gz: 217feaf328809b259c6eb716e968b02417df7b10075a9a1e278c6e337d8a660b
+  data.tar.gz: 351327bd18d5dd3aa7b6a360ebba4a4f574a86b65ed470cffdf15b38e5d16022
 SHA512:
-  metadata.gz: 0bccfd9d20535f856b4dd115eb9126efc2cf2cfbe02d52fcb1f2e14db43747ee90de71f87873924540ea973f1d077d4618b87beb053e09e4d1c52d7b1b33bcc6
-  data.tar.gz: 3d9a7ce81fbb624f4af5e42dcfef10e4059073523a5b410af67575cffb11bf7e243f11d66cc6bf87851ea5e5dd0f2f7d76100923d53cf386df82ede3beb0c3ad
+  metadata.gz: c374adb923d9533e50c1c11355b304ba04ca92112dee36dcacd886d53be0ec910481559c7fdb0f6cdb3b61c7c949b5b917c3540d4bbd2789a9d568015cb52cbb
+  data.tar.gz: 11259ac1ee5e4dfe0de5b18336fd2167b40b8c10c381a283068557da1d206e06c03d5d76ef46411a8803b00c333370f47c21fbfdf4331e528cb05a145803a9b4

data/History.md

@@ -2,6 +2,13 @@
 
 ### [Unreleased]
 
+### 1.0.3
+ - Configurable unrecoverable record types (#40)
+ - Handle exceptions on retrieving AWS credentials (#39)
+ - Suppress emit error label events (#38)
+ - Provide suppress_type_name parameter (#29)
+ - Honor @time_key (data streams) (#28)
+
 ### 1.0.2
  - Honor @hosts parameter for Data Streams (#21)
  - Use template_file for Data Streams (#20)

data/README.OpenSearchInput.md

@@ -17,6 +17,7 @@
   + [reload_on_failure](#reload_on_failure)
   + [resurrect_after](#resurrect_after)
   + [with_transporter_log](#with_transporter_log)
+  + [emit_error_label_event](#emit-error-label-event)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [sniffer_class_name](#sniffer-class-name)
   + [custom_headers](#custom_headers)
@@ -190,6 +191,18 @@ We recommend to set this true if you start to debug this plugin.
 with_transporter_log true
 ```
 
+### emit_error_label_event
+
+Default `emit_error_label_event` value is `true`.
+
+Emitting error label events is default behavior.
+
+When using the followin configuration, OpenSearch plugin will cut error events on error handler:
+
+```aconf
+emit_error_label_event false
+```
+
 ### Client/host certificate options
 
 Need to verify OpenSearch's certificate?  You can use the following parameter to specify a CA instead of using an environment variable.

data/README.md

@@ -76,6 +76,8 @@ Send your logs to OpenSearch (and search them with OpenSearch Dashboards maybe?)
   + [reload_after](#reload-after)
   + [validate_client_version](#validate-client-version)
   + [unrecoverable_error_types](#unrecoverable-error-types)
+  + [unrecoverable_record_types](#unrecoverable-record-types)
+  + [emit_error_label_event](#emit-error-label-event)
   + [verify os version at startup](#verify_os_version_at_startup)
   + [default_opensearch_version](#default_opensearch_version)
   + [custom_headers](#custom_headers)
@@ -356,6 +358,15 @@ utc_index true
 
 By default, the records inserted into index `logstash-YYMMDD` with UTC (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
 
+
+### suppress_type_name
+
+If OpenSearch cluster complains types removal warnings, this can be suppressed with:
+
+```
+suppress_type_name true
+```
+
 ### target_index_key
 
 Tell this plugin to find the index name to write to in the record under this key in preference to other mechanisms. Key can be specified as path to nested record using dot ('.') as a separator.
@@ -1097,6 +1108,33 @@ Then, remove `rejected_execution_exception` from `unrecoverable_error_types` par
 unrecoverable_error_types ["out_of_memory_error"]
 ```
 
+
+### Unrecoverable Record Types
+
+Default `unrecoverable_record_types` parameter is set up loosely.
+Because `json_parse_exception` is caused by invalid JSON record.
+Another possible unrecoverable record error should be included within this paramater.
+
+If you want to handle `security_exception` as _unrecoverable exceptions_, please consider to change `unrecoverable_record_types` parameter from default value:
+
+```
+unrecoverable_record_types ["json_parse_exception", "security_exception"]
+```
+
+If this error type is included in OpenSearch response, an invalid record should be sent in `@ERROR` data pipeline.
+
+### emit_error_label_event
+
+Default `emit_error_label_event` value is `true`.
+
+Emitting error label events is default behavior.
+
+When using the followin configuration, OpenSearch plugin will cut error events on error handler:
+
+```aconf
+emit_error_label_event false
+```
+
 ### verify_os_version_at_startup
 
 Because OpenSearch plugin will ought to change behavior each of OpenSearch major versions.

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.0.2'
+  s.version       = '1.0.3'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/in_opensearch.rb

@@ -73,6 +73,7 @@ module Fluent::Plugin
     config_param :ca_file, :string, :default => nil
     config_param :ssl_version, :enum, list: [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2], :default => :TLSv1_2
     config_param :with_transporter_log, :bool, :default => false
+    config_param :emit_error_label_event, :bool, :default => true
     config_param :sniffer_class_name, :string, :default => nil
     config_param :custom_headers, :hash, :default => {}
     config_param :docinfo_fields, :array, :default => ['_index', '_type', '_id']
@@ -180,6 +181,13 @@ module Fluent::Plugin
       }
     end
 
+    def emit_error_label_event(&block)
+      # If `emit_error_label_event` is specified as false, error event emittions are not occurred.
+      if emit_error_label_event
+        block.call
+      end
+    end
+
     def start
       super
 
@@ -224,7 +232,9 @@ module Fluent::Plugin
     def parse_time(value, event_time, tag)
       @timestamp_parser.call(value)
     rescue => e
-      router.emit_error_event(@timestamp_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @timestamp_key_format, 'value' => value}, e)
+      emit_error_label_event do
+        router.emit_error_event(@timestamp_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @timestamp_key_format, 'value' => value}, e)
+      end
       return Time.at(event_time).to_time
     end
 

data/lib/fluent/plugin/opensearch_error_handler.rb

@@ -49,8 +49,12 @@ class Fluent::Plugin::OpenSearchErrorHandler
     unrecoverable_error_types.include?(type)
   end
 
+  def unrecoverable_record_types
+    @plugin.unrecoverable_record_types
+  end
+
   def unrecoverable_record_error?(type)
-    ['json_parse_exception'].include?(type)
+    unrecoverable_record_types.include?(type)
   end
 
   def log_os_400_reason(&block)
@@ -61,6 +65,13 @@ class Fluent::Plugin::OpenSearchErrorHandler
     end
   end
 
+  def emit_error_label_event(&block)
+    # If `emit_error_label_event` is specified as false, error event emittions are not occurred.
+    if @plugin.emit_error_label_event
+      block.call
+    end
+  end
+
   def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
     items = response['items']
     if items.nil? || !items.is_a?(Array)
@@ -127,12 +138,16 @@ class Fluent::Plugin::OpenSearchErrorHandler
             reason += " [reason]: \'#{item[write_operation]['error']['reason']}\'"
           end
         end
-        @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("400 - Rejected by OpenSearch#{reason}"))
+        emit_error_label_event do
+          @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("400 - Rejected by OpenSearch#{reason}"))
+        end
       else
         if item[write_operation]['error'].is_a?(String)
           reason = item[write_operation]['error']
           stats[:errors_block_resp] += 1
-          @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{reason}"))
+          emit_error_label_event do
+            @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{reason}"))
+          end
           next
         elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
@@ -141,7 +156,9 @@ class Fluent::Plugin::OpenSearchErrorHandler
             raise OpenSearchRequestAbortError, "Rejected OpenSearch due to #{type}"
           end
           if unrecoverable_record_error?(type)
-            @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{type}: #{reason}"))
+            emit_error_label_event do
+              @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{type}: #{reason}"))
+            end
             next
           else
             retry_stream.add(time, rawrecord) unless unrecoverable_record_error?(type)
@@ -150,7 +167,9 @@ class Fluent::Plugin::OpenSearchErrorHandler
           # When we don't have a type field, something changed in the API
           # expected return values.
           stats[:errors_bad_resp] += 1
-          @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - No error type provided in the response"))
+          emit_error_label_event do
+            @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - No error type provided in the response"))
+          end
           next
         end
         stats[type] += 1

data/lib/fluent/plugin/out_opensearch.rb

@@ -110,6 +110,7 @@ module Fluent::Plugin
     config_param :logstash_prefix_separator, :string, :default => '-'
     config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
     config_param :utc_index, :bool, :default => true
+    config_param :suppress_type_name, :bool, :default => false
     config_param :index_name, :string, :default => "fluentd"
     config_param :id_key, :string, :default => nil
     config_param :write_operation, :string, :default => "index"
@@ -160,6 +161,8 @@ module Fluent::Plugin
     config_param :validate_client_version, :bool, :default => false
     config_param :prefer_oj_serializer, :bool, :default => false
     config_param :unrecoverable_error_types, :array, :default => ["out_of_memory_error", "rejected_execution_exception"]
+    config_param :unrecoverable_record_types, :array, :default => ["json_parse_exception"]
+    config_param :emit_error_label_event, :bool, :default => true
     config_param :verify_os_version_at_startup, :bool, :default => true
     config_param :default_opensearch_version, :integer, :default => DEFAULT_OPENSEARCH_VERSION
     config_param :log_os_400_reason, :bool, :default => false
@@ -219,8 +222,8 @@ module Fluent::Plugin
         if conf[:assume_role_arn].nil?
           aws_container_credentials_relative_uri = conf[:ecs_container_credentials_relative_uri] || ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
           if aws_container_credentials_relative_uri.nil?
-            credentials = Aws::SharedCredentials.new({retries: 2}).credentials
-            credentials ||= Aws::InstanceProfileCredentials.new.credentials
+            credentials = Aws::SharedCredentials.new({retries: 2}).credentials rescue nil
+            credentials ||= Aws::InstanceProfileCredentials.new.credentials rescue nil
             credentials ||= Aws::ECSCredentials.new.credentials
           else
             credentials = Aws::ECSCredentials.new({
@@ -462,6 +465,13 @@ module Fluent::Plugin
       placeholder_validities.include?(true)
     end
 
+    def emit_error_label_event(&block)
+      # If `emit_error_label_event` is specified as false, error event emittions are not occurred.
+      if @emit_error_label_event
+        block.call
+      end
+    end
+
     def compression
       !(@compression_level == :no_compression)
     end
@@ -578,7 +588,9 @@ module Fluent::Plugin
     def parse_time(value, event_time, tag)
       @time_parser.call(value)
     rescue => e
-      router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
+      emit_error_label_event do
+        router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
+      end
       return Time.at(event_time).to_datetime
     end
 
@@ -870,7 +882,9 @@ module Fluent::Plugin
             end
           end
         rescue => e
-          router.emit_error_event(tag, time, record, e)
+          emit_error_label_event do
+            router.emit_error_event(tag, time, record, e)
+          end
         end
       end
 
@@ -978,8 +992,12 @@ module Fluent::Plugin
         end
       end
 
-      # OpenSearch only supports "_doc".
-      target_type = DEFAULT_TYPE_NAME
+      if @suppress_type_name
+        target_type = nil
+      else
+        # OpenSearch only supports "_doc".
+        target_type = DEFAULT_TYPE_NAME
+      end
 
       meta.clear
       meta["_index".freeze] = target_index

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -191,12 +191,22 @@ module Fluent::Plugin
       tag = chunk.metadata.tag
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
-
         begin
-          record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
+          if record.has_key?(TIMESTAMP_FIELD)
+            rts = record[TIMESTAMP_FIELD]
+            dt = parse_time(rts, time, tag)
+          elsif record.has_key?(@time_key)
+            rts = record[@time_key]
+            dt = parse_time(rts, time, tag)
+          else
+            dt = Time.at(time).to_datetime
+          end
+          record.merge!({"@timestamp" => dt.iso8601(@time_precision)})
           bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
         rescue => e
-          router.emit_error_event(tag, time, record, e)
+          emit_error_label_event do
+            router.emit_error_event(tag, time, record, e)
+          end
         end
       end
 

data/test/plugin/test_opensearch_error_handler.rb

@@ -35,14 +35,18 @@ class TestOpenSearchErrorHandler < Test::Unit::TestCase
     attr_reader :log
     attr_reader :error_events
     attr_accessor :unrecoverable_error_types
+    attr_accessor :unrecoverable_record_types
     attr_accessor :log_os_400_reason
     attr_accessor :write_operation
+    attr_accessor :emit_error_label_event
     def initialize(log, log_os_400_reason = false)
       @log = log
       @write_operation = 'index'
       @error_events = []
       @unrecoverable_error_types = ["out_of_memory_error", "rejected_execution_exception"]
+      @unrecoverable_record_types = ["json_parse_exception"]
       @log_os_400_reason = log_os_400_reason
+      @emit_error_label_event = true
     end
 
     def router
@@ -135,6 +139,44 @@ class TestOpenSearchErrorHandler < Test::Unit::TestCase
     end
   end
 
+  class TEST400ResponseReasonWithoutErrorLog < self
+    def setup
+      Fluent::Test.setup
+      @log_device = Fluent::Test::DummyLogDevice.new
+      dl_opts = {:log_level => ServerEngine::DaemonLogger::DEBUG}
+      logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
+      @log = Fluent::Log.new(logger)
+      @plugin = TestPlugin.new(@log)
+      @handler = Fluent::Plugin::OpenSearchErrorHandler.new(@plugin)
+      @plugin.emit_error_label_event = false
+    end
+
+    def test_400_responses_reason_log
+      records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
+      response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "status" : 400,
+            "error" : {
+              "type"  : "mapper_parsing_exception",
+              "reason" : "failed to parse"
+            }
+          }
+        }
+      ]
+     }))
+      chunk = MockChunk.new(records)
+      dummy_extracted_values = []
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      assert_equal(0, @plugin.error_events.size)
+      assert_true(@plugin.error_events.empty?)
+    end
+  end
+
   class TEST400ResponseReasonNoDebug < self
     def setup
       Fluent::Test.setup
@@ -177,6 +219,45 @@ class TestOpenSearchErrorHandler < Test::Unit::TestCase
     end
   end
 
+  class TEST400ResponseReasonNoDebugAndNoErrorLog < self
+    def setup
+      Fluent::Test.setup
+      @log_device = Fluent::Test::DummyLogDevice.new
+      dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
+      logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
+      @log = Fluent::Log.new(logger)
+      @plugin = TestPlugin.new(@log)
+      @handler = Fluent::Plugin::OpenSearchErrorHandler.new(@plugin)
+      @plugin.log_os_400_reason = true
+      @plugin.emit_error_label_event = false
+    end
+
+    def test_400_responses_reason_log
+      records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
+      response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "status" : 400,
+            "error" : {
+              "type"  : "mapper_parsing_exception",
+              "reason" : "failed to parse"
+            }
+          }
+        }
+      ]
+     }))
+      chunk = MockChunk.new(records)
+      dummy_extracted_values = []
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      assert_equal(0, @plugin.error_events.size)
+      assert_true(@plugin.error_events.empty?)
+    end
+  end
+
   def test_nil_items_responses
     records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
     response = parse_response(%({

data/test/plugin/test_out_opensearch.rb

@@ -2320,23 +2320,11 @@ class OpenSearchOutputTest < Test::Unit::TestCase
   end
 
   data(
-    "OpenSearch default"=> {"os_version" => 1, "_type" => "_doc"},
+    "OpenSearch default"=> {"os_version" => 1, "_type" => "_doc", "suppress_type" => false},
+    "Suppressed type" => {"os_version" => 1, "_type" => nil, "suppress_type" => true},
   )
   def test_writes_to_speficied_type(data)
-    driver('', data["os_version"]).configure("")
-    stub_opensearch
-    stub_opensearch_info
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
-    end
-    assert_equal(data['_type'], index_cmds.first['index']['_type'])
-  end
-
-  data(
-    "OpenSearch 1"=> {"os_version" => 1, "_type" => "_doc"},
-  )
-  def test_writes_to_speficied_type_with_placeholders(data)
-    driver('', data["os_version"]).configure("")
+    driver('', data["os_version"]).configure("suppress_type_name #{data['suppress_type']}")
     stub_opensearch
     stub_opensearch_info
     driver.run(default_tag: 'test') do

data/test/plugin/test_out_opensearch_data_stream.rb

@@ -9,7 +9,7 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
   include FlexMock::TestCase
   include Fluent::Test::Helpers
 
-  attr_accessor :bulk_records
+  attr_accessor :bulk_records, :index_cmds
 
   OPENSEARCH_DATA_STREAM_TYPE = "opensearch_data_stream"
 
@@ -96,12 +96,14 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
       # {"create": {}}\nhttp://localhost:9200/_data_stream/foo_bar
       # {"@timestamp": ...}
       @bulk_records += req.body.split("\n").size / 2
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end
     stub_request(:post, "http://#{url}#{template_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_data_stream/foo_bar
       # {"@timestamp": ...}
       @bulk_records += req.body.split("\n").size / 2
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end
   end
 
@@ -591,4 +593,72 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
 
     assert_equal(4, connection_resets)
   end
+
+  def test_uses_custom_time_key
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl',
+      'time_key' => 'vtm'
+    })
+
+    ts = DateTime.new(2021,2,3).iso8601(9)
+    record = {
+      'vtm' => ts,
+      'message' => 'Sample Record'
+    }
+
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(ts, index_cmds[1]['@timestamp'])
+  end
+
+  def test_uses_custom_time_key_with_format
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl',
+      'time_key' => 'vtm',
+      'time_key_format' => '%Y-%m-%d %H:%M:%S.%N%z'
+    })
+    ts = "2021-02-03 13:14:01.673+02:00"
+    record = {
+      'vtm' => ts,
+      'message' => 'Sample Record'
+    }
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(DateTime.parse(ts).iso8601(9), index_cmds[1]['@timestamp'])
+  end
+
+  def test_record_no_timestamp
+    stub_default
+    stub_bulk_feed
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl'
+    })
+    record = {
+      'message' => 'Sample Record'
+    }
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-17 00:00:00.000000000 Z
+date: 2022-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd