fluent-plugin-cloudwatch-ingest 1.7.0.rc2 → 1.7.0.rc4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bbcd4224819737806362f26aa66a2e57e3193a76
-  data.tar.gz: 9a78d22537a00b9525e4ca7f92eb0c515ac21ab0
+  metadata.gz: 9c42dca008805bb5f68165fb13411d8a8d6485aa
+  data.tar.gz: 8e0ad98f42b5dde7dbdad9d7966b957a72b3754f
 SHA512:
-  metadata.gz: ccbef5e546fe897d07a90ea7195862e668351b2732e704ae1843c751f01048ff14fe753847f6661b9303349ec015bbc5cd5d262b80ada917a3d5cc2e95655162
-  data.tar.gz: 839333d681717ea4e5af0913c9460b3f40b9afb47ae5efbb8a803b3f2ca51aa632d355c2909ac902d666c91b462af6918a645ce01a575cd0c64ba407ff918e61
+  metadata.gz: 661cc11d5233d808792d5d1b9b23a740141277d45f1631a3c05edf24eeab3836d337c3c9d1a00b630c42036ba31cc92f37805ef9b71275403a57f1fb661cbe79
+  data.tar.gz: b38414fb62cce41769e7ab52e7f762f1d1020780ec4d6d02b5afe921a6c466745e5b124756d83bd0c11326a852a81df962a0f4f955dfc6982c782e12b95cdd09

data/README.md

@@ -41,13 +41,14 @@ Or install it yourself as:
   state_file_name /mnt/nfs/cloudwatch.state
   interval 60
   max_log_streams_per_group 50
-  error_interval 5          # Time to wait between error conditions before retry
-  limit_events 10000        # Number of events to fetch in any given iteration
-  event_start_time 0        # Do not fetch events before this time (UNIX epoch, miliseconds)
-  oldest_logs_first false   # When true fetch the oldest logs first
-  drop_blank_events true    # Fluentd may throw an exception if a blank event is emitted
-  telemetry false           # Produce statsd telemetry
-  statsd_endpoint localhost # Endpoint to which telemetry should be sent
+  error_interval 5            # Time to wait between error conditions before retry
+  get_log_events_interval 0.0 # Time to pause between get_log_events to reduce throttle error 
+  limit_events 10000          # Number of events to fetch in any given iteration
+  event_start_time 0          # Do not fetch events before this time (UNIX epoch, miliseconds)
+  oldest_logs_first false     # When true fetch the oldest logs first
+  drop_blank_events true      # Fluentd may throw an exception if a blank event is emitted
+  telemetry false             # Produce statsd telemetry
+  statsd_endpoint localhost   # Endpoint to which telemetry should be sent
   <parse>
     @type cloudwatch_ingest
     expression /^(?<message>.+)$/

data/lib/fluent/plugin/cloudwatch/ingest/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Ingest
-        VERSION = '1.7.0.rc2'.freeze
+        VERSION = '1.7.0.rc4'.freeze
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_ingest.rb

@@ -34,6 +34,8 @@ module Fluent::Plugin
     desc 'Time to pause between error conditions'
     config_param :error_interval, :time, default: 5
     config_param :api_interval, :time, default: nil
+    desc 'Time to pause between get_log_events to reduce throttle error'
+    config_param :get_log_events_interval, :float, default: 0.0
     desc 'Tag to apply to record'
     config_param :tag, :string, default: 'cloudwatch'
     desc 'Enable AWS SDK logging'
@@ -254,16 +256,51 @@ module Fluent::Plugin
 
     def process_stream(group, stream, next_token, start_time, state)
       event_count = 0
+      has_stream_timestamp = true if state.store[group][stream]['timestamp']
 
       metric(:increment, 'api.calls.getlogevents.attempted')
-      response = @aws.get_log_events(
-        log_group_name: group,
-        log_stream_name: stream,
-        next_token: next_token,
-        limit: @limit_events,
-        start_time: start_time,
-        start_from_head: @oldest_logs_first
-      )
+      begin
+        param_next_token = next_token
+        param_start_time = start_time
+        retried = false
+
+        begin
+          log.info('log_group_name' => group, 'log_stream_name' => stream,
+                   'next_token' => param_next_token, 'limit' => @limit_events,
+                   'start_time' => param_start_time,
+                   'start_from_head' => @oldest_logs_first)
+          sleep @get_log_events_interval
+          response = @aws.get_log_events(
+            log_group_name: group,
+            log_stream_name: stream,
+            next_token: param_next_token,
+            limit: @limit_events,
+            start_time: param_start_time,
+            start_from_head: @oldest_logs_first
+          )
+        rescue Aws::CloudWatchLogs::Errors::InvalidParameterException
+          raise if retried
+
+          metric(:increment, 'api.calls.getlogevents.invalid_token')
+          log.error(
+            'cloudwatch token is expired or broken. '\
+                  'trying with timestamp.'
+          )
+          param_next_token = nil
+          param_start_time = state.store[group][stream]['timestamp']
+          param_start_time ||= @event_start_time
+          retried = true
+          retry
+        end
+      rescue Aws::CloudWatchLogs::Errors::ThrottlingException,
+             Aws::CloudWatchLogs::Errors::ServiceUnavailableException,
+             Seahorse::Client::NetworkingError
+        # on temporary error, save the current state to retry next time
+        if has_stream_timestamp
+          state.new_store[group][stream] = state.store[group][stream]
+        end
+        raise # handle as error in run method
+      end
 
       response.events.each do |e|
         begin
@@ -276,8 +313,6 @@ module Fluent::Plugin
 
       log.info("#{event_count} events processed for stream #{stream}")
 
-      has_stream_timestamp = true if state.store[group][stream]['timestamp']
-
       if !has_stream_timestamp && response.events.count.zero?
         # This stream has returned no data ever.
         # In this case, don't save state (token could be an invalid one)
@@ -333,33 +368,6 @@ module Fluent::Plugin
                     @event_start_time,
                     state
                   )
-                rescue Aws::CloudWatchLogs::Errors::InvalidParameterException
-                  metric(:increment, 'api.calls.getlogevents.invalid_token')
-                  log.error(
-                    'cloudwatch token is expired or broken. '\
-                    'trying with timestamp.'
-                  )
-
-                  # try again with timestamp instead of forward token
-                  begin
-                    timestamp = state.store[group][stream]['timestamp']
-                    timestamp ||= @event_start_time
-
-                    event_count += process_stream(group,
-                                                  stream,
-                                                  nil,
-                                                  timestamp,
-                                                  state)
-                  rescue StandardError => boom
-                    log.error(
-                      'Unable to retrieve events for stream '\
-                      "#{stream} in group #{group}: "\
-                      "#{boom.inspect}"\
-                    )
-                    metric(:increment, 'api.calls.getlogevents.failed')
-                    sleep @error_interval
-                    next
-                  end
                 rescue StandardError => boom
                   log.error("Unable to retrieve events for stream #{stream} "\
                             "in group #{group}: #{boom.inspect}")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-ingest
 version: !ruby/object:Gem::Version
-  version: 1.7.0.rc2
+  version: 1.7.0.rc4
 platform: ruby
 authors:
 - Sam Pointer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-12 00:00:00.000000000 Z
+date: 2018-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler