fluent-plugin-cloudwatch-ingest 1.4.0 → 1.5.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 852a0773b4fd72b1ca24faebd4624b5732d3d916
-  data.tar.gz: 9815dd011589740f0af019e450ff1def0682f11a
+  metadata.gz: e03b673f371cfe94c2f16197a40f1ff0e67d8a82
+  data.tar.gz: 826b1d14832d530aaa0a34532383fdeb203aff7d
 SHA512:
-  metadata.gz: 10ab86bf0fde33eb65869c2db408a3a9bc1658807d4d3e6e8e5b7bd35e9f68f81c4e5a2270dc636e7dbadb3bd306dd9f39c683eba7eb05d9b2b8149df3415845
-  data.tar.gz: 2998fe65f16a65b8582fb6b2ec0ca4b0e3ad20262e229d8f0140cd5bfb7d68bbd93b4c72ad00712ffc8f075db7cae32bc968e82dac2d910fee9f72375015574d
+  metadata.gz: d6ad1bebc3432e40916b94a7d125e75af42a3942b44f16191956293682787602076502acbaa914b5fc319406a95bb393e35b490321e21f3d097fca410341e479
+  data.tar.gz: 306b932009d1c270b70128451e2f2cc3b5d2e0b193ce6390ada56abbe4773111715a319ce3934e08228547fc03b830710ea9f5a85f6907b807847f972614b0b2

data/.rubocop.yml

@@ -1,5 +1,11 @@
+AllCops:
+  DisplayCopNames: true
+  DisplayStyleGuide: true
+  TargetRubyVersion: 2.1
 Metrics/MethodLength:
-  Max: 200
+  Enabled: false
+Metrics/BlockLength:
+  Enabled: false
 Style/Documentation:
   Enabled: false
 Style/ClassAndModuleChildren:

data/CHANGELOG.md

@@ -57,3 +57,9 @@ Both of these changes are designed to make debugging ingestion problems from hig
 * Refuse to emit records with a blank (or newline only) message
 * Emit metric `events.emitted.blocked` to expose these alongside logging
 * Add plugin skew time to telemetry optionally emitted from the parser
+
+## 1.5.0
+
+* Limit the number of streams to be processed with each log stream describe call
+* new parameter `max_log_streams_per_group` with a default of 50 (the default value for *limit* on API calls). This can be increased or decreased to limit the throttling of calls to AWS API
+* Bail out of processing if fluentd has been stopped

data/circle.yml

@@ -5,7 +5,7 @@ test:
     - cp pkg/*.gem ${CIRCLE_ARTIFACTS}
 deployment:
   release:
-    tag: /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$/
+    tag: /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?(\.rc[0-9]+)*$/
     owner: sampointer
     commands:
       - bin/deploy

data/fluent-plugin-cloudwatch-ingest.gemspec

@@ -1,5 +1,3 @@
-# coding: utf-8
-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'fluent/plugin/cloudwatch/ingest/version'
@@ -32,7 +30,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'
 
-  spec.add_dependency 'fluentd', '~>0.14.13'
+  spec.add_dependency 'fluentd', '~>0.14.20'
   spec.add_dependency 'aws-sdk', '~>2.10.9'
   spec.add_dependency 'multi_json', '~>1.12'
   spec.add_dependency 'statsd-ruby', '~>1.4.0'

data/lib/fluent/plugin/cloudwatch/ingest/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Ingest
-        VERSION = '1.4.0'.freeze
+        VERSION = '1.5.0.rc1'.freeze
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_ingest.rb

@@ -1,3 +1,4 @@
+# rubocop:disable Metrics/BlockLength
 require 'aws-sdk'
 require 'fluent/config/error'
 require 'fluent/plugin/input'
@@ -27,7 +28,8 @@ module Fluent::Plugin
     desc 'Log group regexp to exclude, despite matching'
     config_param :log_group_exclude_regexp, :string, default: ''
     desc 'State file name'
-    config_param :state_file_name, :string, default: '/var/spool/td-agent/cloudwatch.state' # rubocop:disable LineLength
+    config_param :state_file_name, :string, default: \
+      '/var/spool/td-agent/cloudwatch.state'
     desc 'Fetch logs every interval'
     config_param :interval, :time, default: 60
     desc 'Time to pause between error conditions'
@@ -39,6 +41,8 @@ module Fluent::Plugin
     config_param :aws_logging_enabled, :bool, default: false
     desc 'Limit the number of events fetched in any iteration'
     config_param :limit_events, :integer, default: 10_000
+    desc 'Limit the number of log streams to be processed with each iteration'
+    config_param :max_log_streams_per_group, :integer, default: 50
     desc 'Do not fetch events before this time'
     config_param :event_start_time, :integer, default: 0
     desc 'Fetch the oldest logs first'
@@ -102,6 +106,8 @@ module Fluent::Plugin
 
       @parser = parser_create(conf: parser_config)
       log.info('Configured fluentd-plugin-cloudwatch-ingest')
+
+      @log_streams_next_token = nil
     end
 
     def start
@@ -120,7 +126,8 @@ module Fluent::Plugin
           role_session_name: @sts_session_name
         )
 
-        log.info("Using STS for authentication with source account ARN: #{@sts_arn}, session name: #{@sts_session_name}") # rubocop:disable LineLength
+        log.info('Using STS for authentication with source account ARN: '\
+                 "#{@sts_arn}, session name: #{@sts_session_name}")
       else
         log.info('Using local instance IAM role for authentication')
       end
@@ -139,7 +146,11 @@ module Fluent::Plugin
     def emit(event, log_group_name, log_stream_name)
       @parser.parse(event, log_group_name, log_stream_name) do |time, record|
         if record['message'].chomp.empty? && @drop_blank_events
-          log.warn("Event is blank or contains only a newline, refusing to emit. group: #{log_group_name}, stream: #{log_stream_name}, event: #{event}") # rubocop:disable LineLength
+          log.warn(
+            'Event is blank or contains only a newline, refusing to '\
+            "emit. group: #{log_group_name}, "\
+            "stream: #{log_stream_name}, event: #{event}"\
+          )
           metric(:increment, 'events.emitted.blocked')
           next
         end
@@ -171,7 +182,10 @@ module Fluent::Plugin
           response.log_groups.each do |group|
             if !@log_group_exclude_regexp.empty?
               if regex.match(group.log_group_name)
-                log.info("Excluding log_group #{group.log_group_name} due to log_group_exclude_regexp #{@log_group_exclude_regexp}") # rubocop:disable LineLength
+                log.info(
+                  "Excluding log_group #{group.log_group_name} due to "\
+                  "log_group_exclude_regexp #{@log_group_exclude_regexp}"\
+                )
                 metric(:increment, 'api.calls.describeloggroups.excluded')
               else
                 log_groups << group.log_group_name
@@ -182,7 +196,7 @@ module Fluent::Plugin
           end
           break unless response.next_token
           next_token = response.next_token
-        rescue => boom
+        rescue StandardError => boom
           log.error("Unable to retrieve log groups: #{boom.inspect}")
           metric(:increment, 'api.calls.describeloggroups.failed')
           next_token = nil
@@ -197,35 +211,44 @@ module Fluent::Plugin
 
     def log_streams(log_group_name, log_stream_name_prefix)
       log_streams = []
-      next_token = nil
-      loop do
-        begin
-          metric(:increment, 'api.calls.describelogstreams.attempted')
-          response = if !log_stream_name_prefix.empty?
-                       @aws.describe_log_streams(
-                         log_group_name: log_group_name,
-                         log_stream_name_prefix: log_stream_name_prefix,
-                         next_token: next_token
-                       )
-                     else
-                       @aws.describe_log_streams(
-                         log_group_name: log_group_name,
-                         next_token: next_token
-                       )
-                     end
 
-          response.log_streams.each { |s| log_streams << s.log_stream_name }
-          break unless response.next_token
-          next_token = response.next_token
-        rescue => boom
-          log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom.inspect}") # rubocop:disable LineLength
-          metric(:increment, 'api.calls.describelogstreams.failed')
-          log_streams = []
-          next_token = nil
-          sleep @error_interval
-          retry
+      begin
+        metric(:increment, 'api.calls.describelogstreams.attempted')
+        response = if !log_stream_name_prefix.empty?
+                     @aws.describe_log_streams(
+                       log_group_name: log_group_name,
+                       log_stream_name_prefix: log_stream_name_prefix,
+                       next_token: @log_streams_next_token,
+                       limit: @max_log_streams_per_group
+                     )
+                   else
+                     @aws.describe_log_streams(
+                       log_group_name: log_group_name,
+                       next_token: @log_streams_next_token,
+                       limit: @max_log_streams_per_group
+                     )
+                   end
+
+        response.log_streams.each { |s| log_streams << s.log_stream_name }
+        if log_streams.size == @max_log_streams_per_group
+          @log_streams_next_token = response.next_token
         end
+      rescue StandardError => boom
+        prefix_message = if log_stream_name_prefix.empty?
+                           ''
+                         else
+                           "with stream prefix #{log_stream_name_prefix}"
+                         end
+
+        log.error('Unable to retrieve log streams '\
+                  "for group #{log_group_name} #{prefix_message}: "\
+                  "#{boom.inspect}")
+        metric(:increment, 'api.calls.describelogstreams.failed')
+
+        sleep @error_interval
+        retry
       end
+
       log.info("Found #{log_streams.size} streams for #{log_group_name}")
 
       return log_streams
@@ -248,11 +271,13 @@ module Fluent::Plugin
         begin
           emit(e, group, stream)
           event_count += 1
-        rescue => boom
+        rescue StandardError => boom
           log.error("Failed to emit event #{e}: #{boom.inspect}")
         end
       end
 
+      log.info("#{event_count} events processed for stream #{stream}")
+
       has_stream_timestamp = true if state.store[group][stream]['timestamp']
 
       if !has_stream_timestamp && response.events.count.zero?
@@ -278,7 +303,7 @@ module Fluent::Plugin
       until @finished
         begin
           state = State.new(@state_file_name, log)
-        rescue => boom
+        rescue StandardError => boom
           log.info("Failed lock state. Sleeping for #{@error_interval}: "\
                    "#{boom.inspect}")
           sleep @error_interval
@@ -286,48 +311,69 @@ module Fluent::Plugin
         end
 
         event_count = 0
-
-        # Fetch the streams for each log group
+        # For each log group
         log_groups(@log_group_name_prefix).each do |group|
-          # For each log stream get and emit the events
-          log_streams(group, @log_stream_name_prefix).each do |stream|
-            state.store[group][stream] = {} unless state.store[group][stream]
-
-            log.info("processing stream: #{stream}")
-
-            # See if we have some stored state for this group and stream.
-            # If we have then use the stored forward_token to pick up
-            # from that point. Otherwise start from the start.
-
+          loop do
             begin
-              event_count += process_stream(group, stream,
-                                            state.store[group][stream]['token'],
-                                            @event_start_time, state)
-            rescue Aws::CloudWatchLogs::Errors::InvalidParameterException
-              metric(:increment, 'api.calls.getlogevents.invalid_token')
-              log.error('cloudwatch token is expired or broken. '\
-                        'trying with timestamp.')
-
-              # try again with timestamp instead of forward token
-              begin
-                timestamp = state.store[group][stream]['timestamp']
-                timestamp = @event_start_time unless timestamp
-
-                event_count += process_stream(group, stream,
-                                              nil, timestamp, state)
-              rescue => boom
-                log.error("Unable to retrieve events for stream #{stream} "\
-                          "in group #{group}: #{boom.inspect}") # rubocop:disable all
-                metric(:increment, 'api.calls.getlogevents.failed')
-                sleep @error_interval
-                next
+              # Fetch log group streams and emit the events
+              log_streams(group, @log_stream_name_prefix).each do |stream|
+                unless state.store[group][stream]
+                  state.store[group][stream] = {}
+                end
+
+                log.info("processing stream: #{stream}")
+
+                # See if we have some stored state for this group and stream.
+                # If we have then use the stored forward_token to pick up
+                # from that point. Otherwise start from the start.
+
+                begin
+                  event_count += process_stream(
+                    group,
+                    stream,
+                    state.store[group][stream]['token'],
+                    @event_start_time,
+                    state
+                  )
+                rescue Aws::CloudWatchLogs::Errors::InvalidParameterException
+                  metric(:increment, 'api.calls.getlogevents.invalid_token')
+                  log.error(
+                    'cloudwatch token is expired or broken. '\
+                    'trying with timestamp.'
+                  )
+
+                  # try again with timestamp instead of forward token
+                  begin
+                    timestamp = state.store[group][stream]['timestamp']
+                    timestamp ||= @event_start_time
+
+                    event_count += process_stream(group,
+                                                  stream,
+                                                  nil,
+                                                  timestamp,
+                                                  state)
+                  rescue StandardError => boom
+                    log.error(
+                      'Unable to retrieve events for stream '\
+                      "#{stream} in group #{group}: "\
+                      "#{boom.inspect}"\
+                    )
+                    metric(:increment, 'api.calls.getlogevents.failed')
+                    sleep @error_interval
+                    next
+                  end
+                rescue StandardError => boom
+                  log.error("Unable to retrieve events for stream #{stream} "\
+                            "in group #{group}: #{boom.inspect}")
+                  metric(:increment, 'api.calls.getlogevents.failed')
+                  sleep @error_interval
+                  next
+                end
               end
-            rescue => boom
-              log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom.inspect}") # rubocop:disable LineLength
-              metric(:increment, 'api.calls.getlogevents.failed')
-              sleep @error_interval
-              next
+              # process log streams in batches
+              break if @log_streams_next_token.nil? || @finished
             end
+            log.info("#{event_count} events processed for group #{group}")
           end
         end
 
@@ -335,19 +381,22 @@ module Fluent::Plugin
         begin
           state.save
           state.close
-        rescue => boom
+        rescue StandardError => boom
           log.error("Unable to save state file: #{boom.inspect}")
         end
 
-        if event_count > 0
-          sleep_interval = @interval
+        if !@finished # no need to sleep if it's finished
+          sleep_interval = if event_count > 0
+                             @interval
+                           else
+                             @error_interval # slow down when no events
+                           end
+
+          log.info("Pausing for #{sleep_interval}")
+          sleep sleep_interval
         else
-          sleep_interval = @error_interval # when there is no events, slow down
+          log.info('Stopping fluentd-plugin-cloudwatch-ingest')
         end
-
-        log.info("#{event_count} events processed.")
-        log.info("Pausing for #{sleep_interval}")
-        sleep sleep_interval
       end
     end
 
@@ -368,7 +417,7 @@ module Fluent::Plugin
           begin
             self.statefile = Pathname.new(@filepath).open('w+')
             save
-          rescue => boom
+          rescue StandardError => boom
             @log.error("Unable to create new file #{statefile.path}: "\
                        "#{boom.inspect}")
           end
@@ -378,13 +427,13 @@ module Fluent::Plugin
         # exception if we can't
         @log.info("Obtaining exclusive lock on state file #{statefile.path}")
         lockstatus = statefile.flock(File::LOCK_EX | File::LOCK_NB)
-        raise CloudwatchIngestInput::State::LockFailed if lockstatus == false
+        raise CloudwatchIngestInput::State::LockFailed unless lockstatus
 
         begin
           @store.merge!(Psych.safe_load(statefile.read))
 
           # Migrate old state file
-          @store.each do |_group, streams|
+          @store.each_value do |streams|
             streams.update(streams) do |_name, stream|
               if stream.is_a? String
                 return { 'token' => stream, 'timestamp' => Time.now.to_i }
@@ -394,7 +443,7 @@ module Fluent::Plugin
           end
 
           @log.info("Loaded #{@store.keys.size} groups from #{statefile.path}")
-        rescue
+        rescue StandardError
           statefile.close
           raise
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-ingest
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0.rc1
 platform: ruby
 authors:
 - Sam Pointer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-21 00:00:00.000000000 Z
+date: 2017-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.14.13
+        version: 0.14.20
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.14.13
+        version: 0.14.20
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -163,9 +163,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.12