fluent-plugin-cloudwatch-logs 0.9.2 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a468cfc9693ee888e05050e47bf6fbd44cda3b9331072cbf0fe28cbb60865cb6
-  data.tar.gz: 7cf780a9d6110ce92279377aa8ecc2f2c311081ab4074a6259a46ebbf2e855ed
+  metadata.gz: 3445e1c11e8856669ee57b3557865f0f4944699dd49a93c61e518c3cbe88de9b
+  data.tar.gz: 43453d4dd9eefed4805541de0e69f0715e2caf9d451c705c5ab403a77f85473b
 SHA512:
-  metadata.gz: d093177804546cb66a2595999d5ef493d4250f0f6916cd5f7b4567e433b322f26351f629e2e2a4c333b21094f9507286771a194e24a232106666fce1383deb31
-  data.tar.gz: 75aab7c98fa0c930de114e1dc122b94713df67cb3287a4edb5c0730dbfb8cc8a2540593bbfd3f8110c4f54581d4942ea4baacbe9f1507dc3b24e54ddf372e768
+  metadata.gz: 8fa3558069bba0d7d9ffe98d61357eef71fd05648c5eaedc614fe0b5a0ced11d8a8fb2bb34a43b4dcab7990bcfd07f4540406334a46293b75be84b2c5986af54
+  data.tar.gz: 7e1a3196f98975d0f9a24d0f14b3a49f14e603eefc76533384fe4bc5bec4547ef9e78d8690ea99c282e04ba861c8ca243684da3ee5ffaa12645f6aa59fe1c670

data/.github/workflows/issue-auto-closer.yml

@@ -0,0 +1,12 @@
+name: Autocloser
+on: [issues]
+jobs:
+  autoclose:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Autoclose issues that did not follow issue template
+      uses: roots/issue-closer-action@v1.1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
+        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and CloudWatchLogs plugin versions.*)"

data/README.md

@@ -43,6 +43,46 @@ Create IAM user with a policy like the following:
 }
 ```
 
+More restricted IAM policy for `out_cloudwatch_logs` is:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "logs:PutLogEvents",
+                "logs:CreateLogGroup",
+                "logs:PutRetentionPolicy",
+                "logs:CreateLogStream",
+                "logs:DescribeLogGroups",
+                "logs:DescribeLogStreams"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
+        }
+    ]
+}
+```
+
+Also, more restricted IAM policy for `in_cloudwatch_logs` is:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "logs:GetLogEvents",
+                "logs:DescribeLogStreams"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
+        }
+    ]
+}
+```
+
 ## Authentication
 
 There are several methods to provide authentication credentials.  Be aware that there are various tradeoffs for these methods,
@@ -149,7 +189,7 @@ Fetch sample log from CloudWatch Logs:
 * `remove_log_group_aws_tags_key`: remove field specified by `log_group_aws_tags_key`
 * `remove_log_group_name_key`: remove field specified by `log_group_name_key`
 * `remove_log_stream_name_key`: remove field specified by `log_stream_name_key`
-* `remove_retention_in_days`: remove field specified by `retention_in_days`
+* `remove_retention_in_days_key`: remove field specified by `retention_in_days_key`
 * `retention_in_days`: use to set the expiry time for log group when created with `auto_create_stream`. (default to no expiry)
 * `retention_in_days_key`: use specified field of records as retention period
 * `use_tag_as_group`: to use tag as a group name
@@ -178,6 +218,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
   #<parse>
   # @type none # or csv, tsv, regexp etc.
   #</parse>
+  #<storage>
+  # @type local # or redis, memcached, etc.
+  #</storage>
 </source>
 ```
 
@@ -193,7 +236,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `log_group_name`: name of log group to fetch logs
 * `log_stream_name`: name of log stream to fetch logs
 * `region`: AWS Region.  See [Authentication](#authentication) for more information.
-* `state_file`: file to store current state (e.g. next\_forward\_token)
+* `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
+* `include_metadata`: include metadata such as `log_group_name` and `log_stream_name`. (default: false)
+* `state_file`: file to store current state (e.g. next\_forward\_token). This parameter is deprecated. Use `<storage>` instead.
 * `tag`: fluentd tag
 * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
 * `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
@@ -203,6 +248,7 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `time_range_format`: specify time format for time range. (default: `%Y-%m-%d %H:%M:%S`)
 * `format`: specify CloudWatchLogs' log format. (default `nil`)
 * `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
+* `<storage>`: specify storage plugin configuration. see also: https://docs.fluentd.org/v/1.0/storage#how-to-use
 
 ## Test
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.9.2"
+        VERSION = "0.10.1"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -8,7 +8,9 @@ module Fluent::Plugin
   class CloudwatchLogsInput < Input
     Fluent::Plugin.register_input('cloudwatch_logs', self)
 
-    helpers :parser, :thread, :compat_parameters
+    helpers :parser, :thread, :compat_parameters, :storage
+
+    DEFAULT_STORAGE_TYPE = 'local'
 
     config_param :aws_key_id, :string, default: nil, secret: true
     config_param :aws_sec_key, :string, default: nil, secret: true
@@ -21,7 +23,8 @@ module Fluent::Plugin
     config_param :log_group_name, :string
     config_param :log_stream_name, :string, default: nil
     config_param :use_log_stream_name_prefix, :bool, default: false
-    config_param :state_file, :string
+    config_param :state_file, :string, default: nil,
+                 deprecated: "Use <stroage> instead."
     config_param :fetch_interval, :time, default: 60
     config_param :http_proxy, :string, default: nil
     config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
@@ -30,11 +33,19 @@ module Fluent::Plugin
     config_param :start_time, :string, default: nil
     config_param :end_time, :string, default: nil
     config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
+    config_param :throttling_retry_seconds, :time, default: nil
+    config_param :include_metadata, :bool, default: false
 
     config_section :parse do
       config_set_default :@type, 'none'
     end
 
+    config_section :storage do
+      config_set_default :usage, 'store_next_tokens'
+      config_set_default :@type, DEFAULT_STORAGE_TYPE
+      config_set_default :persistent, false
+    end
+
     def initialize
       super
 
@@ -52,6 +63,7 @@ module Fluent::Plugin
       if @start_time && @end_time && (@end_time < @start_time)
         raise Fluent::ConfigError, "end_time(#{@end_time}) should be greater than start_time(#{@start_time})."
       end
+      @next_token_storage = storage_create(usage: 'store_next_tokens', conf: config, default_type: DEFAULT_STORAGE_TYPE)
     end
 
     def start
@@ -98,20 +110,28 @@ module Fluent::Plugin
       end
     end
 
-    def state_file_for(log_stream_name)
-      return "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}" if log_stream_name
-      return @state_file
+    def state_key_for(log_stream_name)
+      if log_stream_name
+        "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
+      else
+        @state_file
+      end
+    end
+
+    def migrate_state_file_to_storage(log_stream_name)
+      @next_token_storage.put(:"#{state_key_for(log_stream_name)}", File.read(state_key_for(log_stream_name)).chomp)
+      File.delete(state_key_for(log_stream_name))
     end
 
     def next_token(log_stream_name)
-      return nil unless File.exist?(state_file_for(log_stream_name))
-      File.read(state_file_for(log_stream_name)).chomp
+      if @next_token_storage.persistent && File.exist?(state_key_for(log_stream_name))
+        migrate_state_file_to_storage(log_stream_name)
+      end
+      @next_token_storage.get(:"#{state_key_for(log_stream_name)}")
     end
 
     def store_next_token(token, log_stream_name = nil)
-      File.open(state_file_for(log_stream_name), 'w') do |f|
-        f.write token
-      end
+      @next_token_storage.put(:"#{state_key_for(log_stream_name)}", token)
     end
 
     def run
@@ -129,8 +149,16 @@ module Fluent::Plugin
               log_streams.each do |log_stream|
                 log_stream_name = log_stream.log_stream_name
                 events = get_events(log_stream_name)
+                metadata = if @include_metadata
+                             {
+                               "log_stream_name" => log_stream_name,
+                               "log_group_name" => @log_group_name
+                             }
+                           else
+                             {}
+                           end
                 events.each do |event|
-                  emit(log_stream_name, event)
+                  emit(log_stream_name, event, metadata)
                 end
               end
             rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
@@ -139,8 +167,16 @@ module Fluent::Plugin
             end
           else
             events = get_events(@log_stream_name)
+            metadata = if @include_metadata
+                          {
+                            "log_stream_name" => @log_stream_name,
+                            "log_group_name" => @log_group_name
+                          }
+                        else
+                          {}
+                        end
             events.each do |event|
-              emit(log_stream_name, event)
+              emit(log_stream_name, event, metadata)
             end
           end
         end
@@ -148,18 +184,24 @@ module Fluent::Plugin
       end
     end
 
-    def emit(stream, event)
+    def emit(stream, event, metadata)
       if @parser
         @parser.parse(event.message) {|time,record|
           if @use_aws_timestamp
             time = (event.timestamp / 1000).floor
           end
+          unless metadata.empty?
+            record.merge!("metadata" => metadata)
+          end
           router.emit(@tag, time, record)
         }
       else
         time = (event.timestamp / 1000).floor
         begin
           record = @json_handler.load(event.message)
+          unless metadata.empty?
+            record.merge!("metadata" => metadata)
+          end
           router.emit(@tag, time, record)
         rescue JSON::ParserError, Yajl::ParseError => error # Catch parser errors
           log.error "Invalid JSON encountered while parsing event.message"
@@ -169,38 +211,55 @@ module Fluent::Plugin
     end
 
     def get_events(log_stream_name)
-      request = {
-        log_group_name: @log_group_name,
-        log_stream_name: log_stream_name
-      }
-      request.merge!(start_time: @start_time) if @start_time
-      request.merge!(end_time: @end_time) if @end_time
-      log_next_token = next_token(log_stream_name)
-      request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
-      response = @logs.get_log_events(request)
-      if valid_next_token(log_next_token, response.next_forward_token)
-        store_next_token(response.next_forward_token, log_stream_name)
-      end
+      throttling_handler('get_log_events') do
+        request = {
+          log_group_name: @log_group_name,
+          log_stream_name: log_stream_name
+        }
+        request.merge!(start_time: @start_time) if @start_time
+        request.merge!(end_time: @end_time) if @end_time
+        log_next_token = next_token(log_stream_name)
+        request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
+        response = @logs.get_log_events(request)
+        if valid_next_token(log_next_token, response.next_forward_token)
+          store_next_token(response.next_forward_token, log_stream_name)
+        end
 
-      response.events
+        response.events
+      end
     end
 
     def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
-      request = {
-        log_group_name: @log_group_name
-      }
-      request[:next_token] = next_token if next_token
-      request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
-      response = @logs.describe_log_streams(request)
-      if log_streams
-        log_streams.concat(response.log_streams)
-      else
-        log_streams = response.log_streams
+      throttling_handler('describe_log_streams') do
+        request = {
+          log_group_name: @log_group_name
+        }
+        request[:next_token] = next_token if next_token
+        request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
+        response = @logs.describe_log_streams(request)
+        if log_streams
+          log_streams.concat(response.log_streams)
+        else
+          log_streams = response.log_streams
+        end
+        if response.next_token
+          log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
+        end
+        log_streams
       end
-      if response.next_token
-        log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
+    end
+
+    def throttling_handler(method_name)
+      yield
+    rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
+      if throttling_retry_seconds
+        log.warn "ThrottlingException #{method_name}. Waiting #{throttling_retry_seconds} seconds to retry."
+        sleep throttling_retry_seconds
+
+        throttling_handler(method_name) { yield }
+      else
+        raise err
       end
-      log_streams
     end
 
     def valid_next_token(prev_token, next_token)

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -7,6 +7,8 @@ module Fluent::Plugin
   class CloudwatchLogsOutput < Output
     Fluent::Plugin.register_output('cloudwatch_logs', self)
 
+    class TooLargeEventError < Fluent::UnrecoverableError; end
+
     helpers :compat_parameters, :inject
 
     DEFAULT_BUFFER_TYPE = "memory"
@@ -41,7 +43,7 @@ module Fluent::Plugin
     config_param :remove_log_group_aws_tags_key, :bool, default: false
     config_param :retention_in_days, :integer, default: nil
     config_param :retention_in_days_key, :string, default: nil
-    config_param :remove_retention_in_days, :bool, default: false
+    config_param :remove_retention_in_days_key, :bool, default: false
     config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
     config_param :log_rejected_request, :bool, :default => false
 
@@ -219,6 +221,14 @@ module Fluent::Plugin
 
         events = []
         rs.each do |t, time, record|
+          if @log_group_aws_tags_key && @remove_log_group_aws_tags_key
+            record.delete(@log_group_aws_tags_key)
+          end
+
+          if @retention_in_days_key && @remove_retention_in_days_key
+            record.delete(@retention_in_days_key)
+          end
+
           record = drop_empty_record(record)
 
           time_ms = (time.to_f * 1000).floor
@@ -311,8 +321,7 @@ module Fluent::Plugin
       while event = events.shift
         event_bytesize = event[:message].bytesize + EVENT_HEADER_SIZE
         if MAX_EVENT_SIZE < event_bytesize
-          log.warn "Log event in #{group_name} is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
-          break
+          raise TooLargeEventError, "Log event in #{group_name} is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
         end
 
         new_chunk = chunk + [event]
@@ -369,8 +378,7 @@ module Fluent::Plugin
           end
         rescue Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException, Aws::CloudWatchLogs::Errors::DataAlreadyAcceptedException => err
           sleep 1 # to avoid too many API calls
-          log_stream = find_log_stream(group_name, stream_name)
-          store_next_sequence_token(group_name, stream_name, log_stream.upload_sequence_token)
+          store_next_sequence_token(group_name, stream_name, err.expected_sequence_token)
           log.warn "updating upload sequence token forcefully because unrecoverable error occured", {
             "error" => err,
             "log_group" => group_name,
@@ -392,7 +400,13 @@ module Fluent::Plugin
             raise err
           end
         rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
-          if !@put_log_events_disable_retry_limit && @put_log_events_retry_limit < retry_count
+          if @put_log_events_retry_limit < 1
+            log.warn "failed to PutLogEvents and discard logs because put_log_events_retry_limit is less than 1", {
+              "error_class" => err.class.to_s,
+              "error" => err.message,
+            }
+            return
+          elsif !@put_log_events_disable_retry_limit && @put_log_events_retry_limit < retry_count
             log.error "failed to PutLogEvents and discard logs because retry count exceeded put_log_events_retry_limit", {
               "error_class" => err.class.to_s,
               "error" => err.message,

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -28,6 +28,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
         start_time "2019-06-18 00:00:00Z"
         end_time "2020-01-18 00:00:00Z"
         time_range_format "%Y-%m-%d %H:%M:%S%z"
+        throttling_retry_seconds 30
       EOC
 
       assert_equal('test_id', d.instance.aws_key_id)
@@ -43,6 +44,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(1560816000000, d.instance.start_time)
       assert_equal(1579305600000, d.instance.end_time)
       assert_equal("%Y-%m-%d %H:%M:%S%z", d.instance.time_range_format)
+      assert_equal(30, d.instance.throttling_retry_seconds)
     end
 
     test 'invalid time range' do
@@ -97,6 +99,34 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
     end
 
+    def test_emit_with_metadata
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      put_log_events([
+        {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+        {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+      ])
+
+      sleep 5
+
+      d = create_driver(default_config + %[include_metadata true])
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_true(emits[0][2].has_key?("metadata"))
+      assert_true(emits[1][2].has_key?("metadata"))
+      emits[0][2].delete_if {|k, v|
+        k == "metadata"
+      }
+      emits[1][2].delete_if {|k, v|
+        k == "metadata"
+      }
+      assert_equal(2, emits.size)
+      assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+      assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+    end
+
     def test_emit_with_aws_timestamp
       create_log_stream
 
@@ -171,7 +201,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
                            '@type' => 'cloudwatch_logs',
                            'log_group_name' => "#{log_group_name}",
                            'log_stream_name' => "#{log_stream_name}",
-                           'state_file' => '/tmp/state',
                           }
       cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
       cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
@@ -181,7 +210,49 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       csv_format_config = config_element('ROOT', '', cloudwatch_config, [
                                            config_element('parse', '', {'@type' => 'csv',
                                                                         'keys' => 'time,message',
-                                                                        'time_key' => 'time'})
+                                                                        'time_key' => 'time'}),
+                                           config_element('storage', '', {'@type' => 'local',
+                                                                          'path' => '/tmp/state'})
+                                         ])
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      log_time_ms = time_ms - 10000
+      put_log_events([
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
+      ])
+
+      sleep 5
+
+      d = create_driver(csv_format_config)
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_equal(2, emits.size)
+      assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
+      assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
+    end
+
+    test "emit with <parse> csv with metadata" do
+      cloudwatch_config = {'tag' => "test",
+                           '@type' => 'cloudwatch_logs',
+                           'log_group_name' => "#{log_group_name}",
+                           'log_stream_name' => "#{log_stream_name}",
+                           'include_metadata' => true,
+                          }
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
+
+      csv_format_config = config_element('ROOT', '', cloudwatch_config, [
+                                           config_element('parse', '', {'@type' => 'csv',
+                                                                        'keys' => 'time,message',
+                                                                        'time_key' => 'time'}),
+                                           config_element('storage', '', {'@type' => 'local',
+                                                                          'path' => '/tmp/state'})
+
                                          ])
       create_log_stream
 
@@ -198,6 +269,14 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       d.run(expect_emits: 2, timeout: 5)
 
       emits = d.events
+      assert_true(emits[0][2].has_key?("metadata"))
+      assert_true(emits[1][2].has_key?("metadata"))
+      emits[0][2].delete_if {|k, v|
+        k == "metadata"
+      }
+      emits[1][2].delete_if {|k, v|
+        k == "metadata"
+      }
       assert_equal(2, emits.size)
       assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
       assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
@@ -244,7 +323,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
                            '@type' => 'cloudwatch_logs',
                            'log_group_name' => "#{log_group_name}",
                            'log_stream_name' => "#{log_stream_name}",
-                           'state_file' => '/tmp/state',
                           }
       cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
       cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
@@ -254,7 +332,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       regex_format_config = config_element('ROOT', '', cloudwatch_config, [
                                            config_element('parse', '', {'@type' => 'regexp',
                                                                         'expression' => "/^(?<cloudwatch>[^ ]*)?/",
-                                                                       })
+                                                                       }),
+                                           config_element('storage', '', {'@type' => 'local',
+                                                                          'path' => '/tmp/state'})
                                          ])
       create_log_stream
 
@@ -608,6 +688,59 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(["test", ((time_ms + 7000) / 1000), { "cloudwatch" => "logs7" }], events[6])
       assert_equal(["test", ((time_ms + 8000) / 1000), { "cloudwatch" => "logs8" }], events[7])
     end
+
+    test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException in get_log_events" do
+      config = <<-CONFIG
+        tag test
+        @type cloudwatch_logs
+        log_group_name #{log_group_name}
+        state_file /tmp/state
+        fetch_interval 0.1
+        throttling_retry_seconds 0.2
+      CONFIG
+
+      # it will raises the error 2 times
+      counter = 0
+      times = 2
+      stub(@client).get_log_events(anything) {
+        counter += 1
+        counter <= times ? raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error")) : OpenStruct.new(events: [], next_forward_token: nil)
+      }
+
+      d = create_driver(config)
+
+      # so, it is expected to valid_next_token once
+      mock(d.instance).valid_next_token(nil, nil).once
+
+      d.run
+      assert_equal(2, d.logs.select {|l| l =~ /ThrottlingException get_log_events. Waiting 0.2 seconds to retry/ }.size)
+    end
+
+    test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException in describe_log_streams" do
+      config = <<-CONFIG
+        tag test
+        @type cloudwatch_logs
+        log_group_name #{log_group_name}
+        use_log_stream_name_prefix true
+        state_file /tmp/state
+        fetch_interval 0.1
+        throttling_retry_seconds 0.2
+      CONFIG
+
+      # it will raises the error 2 times
+      log_stream = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
+      counter = 0
+      times = 2
+      stub(@client).describe_log_streams(anything) {
+        counter += 1
+        counter <= times ? raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error")) : OpenStruct.new(log_streams: [log_stream], next_token: nil)
+      }
+
+      d = create_driver(config)
+
+      d.run
+      assert_equal(2, d.logs.select {|l| l =~ /ThrottlingException describe_log_streams. Waiting 0.2 seconds to retry/ }.size)
+    end
   end
 
   private

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -547,6 +547,37 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert(d.logs.any?{|log| log.include?("failed to set retention policy for Log group")})
     end
 
+    def test_remove_retention_in_days_key
+      new_log_stream
+
+      d = create_driver(<<-EOC)
+        #{default_config}
+        log_group_name #{log_group_name}
+        log_stream_name #{log_stream_name}
+        retention_in_days_key retention_in_days
+        remove_retention_in_days_key true
+      EOC
+
+      records = [
+        {'cloudwatch' => 'logs1', 'message' => 'message1', 'retention_in_days' => '7'},
+        {'cloudwatch' => 'logs2', 'message' => 'message2', 'retention_in_days' => '7'},
+      ]
+
+      time = Time.now
+      d.run(default_tag: fluentd_tag) do
+        records.each_with_index do |record, i|
+          d.feed(time.to_i + i, record)
+        end
+      end
+
+      sleep 10
+
+      events = get_log_events
+      assert_equal(2, events.size)
+      assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
+      assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
+    end
+
     def test_log_group_aws_tags_key
       clear_log_group
 
@@ -576,6 +607,37 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert_equal("value2", awstags.fetch("tag2"))
     end
 
+    def test_remove_log_group_aws_tags_key
+      new_log_stream
+
+      d = create_driver(<<-EOC)
+        #{default_config}
+        log_group_name #{log_group_name}
+        log_stream_name #{log_stream_name}
+        log_group_aws_tags_key log_group_tags
+        remove_log_group_aws_tags_key true
+      EOC
+
+      records = [
+        {'cloudwatch' => 'logs1', 'message' => 'message1', 'log_group_tags' => {"tag1" => "value1", "tag2" => "value2"}},
+        {'cloudwatch' => 'logs2', 'message' => 'message2', 'log_group_tags' => {"tag1" => "value1", "tag2" => "value2"}},
+      ]
+
+      time = Time.now
+      d.run(default_tag: fluentd_tag) do
+        records.each_with_index do |record, i|
+          d.feed(time.to_i + i, record)
+        end
+      end
+
+      sleep 10
+
+      events = get_log_events
+      assert_equal(2, events.size)
+      assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
+      assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
+    end
+
     def test_log_group_aws_tags_key_same_group_diff_tags
       clear_log_group
 
@@ -651,6 +713,32 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
     end
 
+    def test_retrying_on_throttling_exception_with_put_log_events_retry_limit_as_zero
+      client = Aws::CloudWatchLogs::Client.new
+      @called = false
+      stub(client).put_log_events(anything) {
+        raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error"))
+      }.once.ordered
+
+      d = create_driver(<<-EOC)
+        #{default_config}
+        log_group_name #{log_group_name}
+        log_stream_name #{log_stream_name}
+        @log_level debug
+        put_log_events_retry_limit 0
+      EOC
+      time = event_time
+      d.instance.instance_variable_set(:@logs, client)
+      d.run(default_tag: fluentd_tag) do
+        d.feed(time, {'message' => 'message1'})
+      end
+
+      logs = d.logs
+      assert_equal(0, logs.select {|l| l =~ /Called PutLogEvents API/ }.size)
+      assert_equal(1, logs.select {|l| l =~ /failed to PutLogEvents/ }.size)
+      assert_equal(0, logs.select {|l| l =~ /retry succeeded/ }.size)
+    end
+
     def test_retrying_on_throttling_exception
       resp = Object.new
       mock(resp).rejected_log_events_info {}

data/test/test_helper.rb

@@ -42,10 +42,7 @@ module CloudwatchLogsTestHelper
   end
 
   def log_stream_name(log_stream_name_prefix = nil)
-    if !@log_stream_name
-      new_log_stream(log_stream_name_prefix)
-    end
-    @log_stream_name
+    @log_stream_name ||= new_log_stream(log_stream_name_prefix)
   end
 
   def new_log_stream(log_stream_name_prefix = nil)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.10.1
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-02 00:00:00.000000000 Z
+date: 2020-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -115,6 +115,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/issue-auto-closer.yml"
 - ".gitignore"
 - ".travis.yml"
 - Gemfile