fluent-plugin-cloudwatch-ingest 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fb407b315654aac2dbac07280a22beb45dca1358
-  data.tar.gz: 15798196f5d8653346d63e037e3585992d486ae6
+  metadata.gz: 867b3c94f91c292c250bb5b2071a6f0115f56051
+  data.tar.gz: 2844dc5a9d0443f15f5b240ca1c91b84b35b512c
 SHA512:
-  metadata.gz: 68a1ada252e002cd072928bbb22f7a0caaa39b70b40a6b742d555d306fb3efffbd6887c439b10d47efaa0675315eb2a7cf3993468c1b44b402757aaf0f3712b4
-  data.tar.gz: a5a87902b86858f5bc88c151fabbde3af46591f3ba3251363c1dd824bc072775cf77681416a3b4c61dc47bdc40e51bc5dcec6a2f5915d36dbba79f6bc8a8a814
+  metadata.gz: 7b8a46269c09a36d7b1918f9e12ff01fec05a57366aad780a36589cec423df5d5d77602edba436e771e507164506ea8b54b9f1b6fe20c89d4eb9b19e91e88120
+  data.tar.gz: d9c71cf85ca8cafad29ac04ab5a9ef35eab1a47f245983b33e05a301019a39cb1db4f78fa4c3954fb0b4b2351dd1aefc1e7eed015e78dac79190a0a4fe8335cc

data/.rubocop.yml

@@ -12,3 +12,5 @@ Metrics/AbcSize:
   Enabled: false
 Metrics/CyclomaticComplexity:
   Max: 10
+Metrics/ClassLength:
+  Enabled: false

data/lib/fluent/plugin/cloudwatch/ingest/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Ingest
-        VERSION = '0.1.9'.freeze
+        VERSION = '0.1.10'.freeze
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_ingest.rb

@@ -28,206 +28,206 @@ module Fluent::Plugin
     config_param :interval, :time, default: 60
     desc 'Time to pause between API call failures'
     config_param :api_interval, :time, default: 120
-  end
-
-  def initialize
-    super
-    log.info('Starting fluentd-plugin-cloudwatch-ingest')
-  end
 
-  def configure(conf)
-    super
-    log.info('Configured fluentd-plugin-cloudwatch-ingest')
-  end
-
-  def start
-    super
-    log.info('Started fluentd-plugin-cloudwatch-ingest')
-
-    # Get a handle to Cloudwatch
-    aws_options = {}
-    Aws.config[:region] = @region
-    log.info("Working in region #{@region}")
-
-    if @sts_enabled
-      aws_options[:credentials] = Aws::AssumeRoleCredentials.new(
-        role_arn: @sts_arn,
-        role_session_name: @sts_session_name
-      )
-
-      log.info("Using STS for authentication with source account ARN:
-        #{@sts_arn}, session name: #{@sts_session_name}")
-    else
-      log.info('Using local instance IAM role for authentication')
+    def initialize
+      super
+      log.info('Starting fluentd-plugin-cloudwatch-ingest')
     end
-    @aws = Aws::CloudWatchLogs::Client.new(aws_options)
-    @finished = false
-    @thread = Thread.new(&method(:run))
-  end
 
-  def shutdown
-    @finished = true
-    @thread.join
-  end
-
-  private
+    def configure(conf)
+      super
+      log.info('Configured fluentd-plugin-cloudwatch-ingest')
+    end
 
-  def emit(log_event)
-    # TODO: I need to do something useful
-  end
+    def start
+      super
+      log.info('Started fluentd-plugin-cloudwatch-ingest')
 
-  def log_groups(log_group_prefix)
-    log_groups = []
+      # Get a handle to Cloudwatch
+      aws_options = {}
+      Aws.config[:region] = @region
+      log.info("Working in region #{@region}")
 
-    # Fetch all log group names
-    next_token = nil
-    loop do
-      begin
-        response = @aws.describe_log_groups(
-          log_group_name_prefix: log_group_prefix,
-          next_token: next_token
+      if @sts_enabled
+        aws_options[:credentials] = Aws::AssumeRoleCredentials.new(
+          role_arn: @sts_arn,
+          role_session_name: @sts_session_name
         )
 
-        response.log_groups.each { |g| log_groups << g.log_group_name }
-        break unless response.next_token
-        next_token = response.next_token
-      rescue => boom
-        log.error("Unable to retrieve log groups: #{boom}")
-        next_token = nil
-        sleep @api_interval
-        retry
+        log.info("Using STS for authentication with source account ARN:
+          #{@sts_arn}, session name: #{@sts_session_name}")
+      else
+        log.info('Using local instance IAM role for authentication')
       end
+      @aws = Aws::CloudWatchLogs::Client.new(aws_options)
+      @finished = false
+      @thread = Thread.new(&method(:run))
     end
-    log.info("Found #{log_groups.size} log groups")
 
-    return log_groups
-  end
+    def shutdown
+      @finished = true
+      @thread.join
+    end
 
-  def log_streams(log_group_name, log_stream_name_prefix)
-    log_streams = []
-    next_token = nil
-    loop do
-      begin
-        response = @aws.describe_log_streams(
-          log_group_name: group,
-          log_stream_name_prefix: log_stream_name_prefix,
-          next_token: next_token
-        )
+    private
 
-        response.log_streams.each { |s| log_streams << s.log_stream_name }
-        break unless reponse.next_token
-        next_token = reponse.next_token
-      rescue => boom
-        log.error("Unable to retrieve log streams for group #{group}
-          with stream prefix #{log_stream_name_prefix}: #{boom}")
-        log_streams = []
-        next_token = nil
-        sleep @api_interval
-        retry
-      end
+    def emit(log_event)
+      # TODO: I need to do something useful
     end
-    log.info("Found #{log_streams.size} streams for #{log_group_name}")
 
-    return log_streams
-  end
+    def log_groups(log_group_prefix)
+      log_groups = []
 
-  def run
-    until @finished
-      begin
-        state = State.new(@state_file_name)
-      rescue => boom
-        log.info("Failed to get state lock. Sleeping for #{@interval}: #{boom}")
-        sleep @interval
-        retry
+      # Fetch all log group names
+      next_token = nil
+      loop do
+        begin
+          response = @aws.describe_log_groups(
+            log_group_name_prefix: log_group_prefix,
+            next_token: next_token
+          )
+
+          response.log_groups.each { |g| log_groups << g.log_group_name }
+          break unless response.next_token
+          next_token = response.next_token
+        rescue => boom
+          log.error("Unable to retrieve log groups: #{boom}")
+          next_token = nil
+          sleep @api_interval
+          retry
+        end
       end
+      log.info("Found #{log_groups.size} log groups")
 
-      # Fetch the streams for each log group
-      log_groups(@log_group_name_prefix).each do |group|
-        # For each log stream get and emit the events
-        log_streams(group, @log_stream_name_prefix).each do |stream|
-          # See if we have some stored state for this group and stream.
-          # If we have then use the stored forward_token to pick up
-          # from that point. Otherwise start from the start.
-          stream_token = (state[group][stream] if state[group][stream])
-
-          begin
-            loop do
-              response = @aws.get_log_events(
-                log_group_name: group,
-                log_stream_name: stream,
-                next_token: stream_token
-              )
-
-              emit(response.events)
-              break unless response.next_token
-              stream_token = response.next_token
-            end
+      return log_groups
+    end
 
-            # Once all events for this stream have been processed,
-            # store the forward token
-            state[group][stream] = response.next_forward_token
-          rescue => boom
-            log.error("Unable to retrieve events for stream
-              #{stream} in group #{group}: #{boom}")
-            sleep @api_interval
-            retry
-          end
+    def log_streams(log_group_name, log_stream_name_prefix)
+      log_streams = []
+      next_token = nil
+      loop do
+        begin
+          response = @aws.describe_log_streams(
+            log_group_name: group,
+            log_stream_name_prefix: log_stream_name_prefix,
+            next_token: next_token
+          )
+
+          response.log_streams.each { |s| log_streams << s.log_stream_name }
+          break unless reponse.next_token
+          next_token = reponse.next_token
+        rescue => boom
+          log.error("Unable to retrieve log streams for group #{group}
+            with stream prefix #{log_stream_name_prefix}: #{boom}")
+          log_streams = []
+          next_token = nil
+          sleep @api_interval
+          retry
         end
       end
+      log.info("Found #{log_streams.size} streams for #{log_group_name}")
 
-      log.info('Pruning and saving state')
-      state.prune(log_groups) # Remove dead streams
-      begin
-        state.save
-        state.close
-      rescue
-        log.error("Unable to save state file: #{boom}")
-      end
-      log.info("Pausing for #{@interval}")
-      sleep @interval
+      return log_streams
     end
-  end
-
-  class CloudwatchIngestInput::State < Hash
-    class LockFailed < RuntimeError; end
-    attr_accessor :statefile
 
-    def initialize(filepath)
-      self.statefile = Pathname.new(filepath).open('w')
-      unless statefile.exists?
-        log.warn("State file #{statefile} does not exist. Creating a new one.")
+    def run
+      until @finished
         begin
-          save
+          state = State.new(@state_file_name)
         rescue => boom
-          log.error("Unable to create new state file #{statefile}: #{boom}")
+          log.info("Failed lock state. Sleeping for #{@interval}: #{boom}")
+          sleep @interval
+          retry
         end
-      end
 
-      # Attempt to obtain an exclusive flock on the file and raise and
-      # exception if we can't
-      log.info("Obtaining exclusive lock on state file #{statefile}")
-      lockstatus = statefile.flock(File::LOCK_EX | File::LOCK_NB)
-      raise CloudwatchIngestInput::State::LockFailed if lockstatus == false
+        # Fetch the streams for each log group
+        log_groups(@log_group_name_prefix).each do |group|
+          # For each log stream get and emit the events
+          log_streams(group, @log_stream_name_prefix).each do |stream|
+            # See if we have some stored state for this group and stream.
+            # If we have then use the stored forward_token to pick up
+            # from that point. Otherwise start from the start.
+            stream_token = (state[group][stream] if state[group][stream])
+
+            begin
+              loop do
+                response = @aws.get_log_events(
+                  log_group_name: group,
+                  log_stream_name: stream,
+                  next_token: stream_token
+                )
+
+                emit(response.events)
+                break unless response.next_token
+                stream_token = response.next_token
+              end
+
+              # Once all events for this stream have been processed,
+              # store the forward token
+              state[group][stream] = response.next_forward_token
+            rescue => boom
+              log.error("Unable to retrieve events for stream
+                #{stream} in group #{group}: #{boom}")
+              sleep @api_interval
+              retry
+            end
+          end
+        end
 
-      merge!(YAML.safe_load(statefile.read))
-      log.info("Loaded state for #{keys.size} log groups from #{statefile}")
+        log.info('Pruning and saving state')
+        state.prune(log_groups) # Remove dead streams
+        begin
+          state.save
+          state.close
+        rescue
+          log.error("Unable to save state file: #{boom}")
+        end
+        log.info("Pausing for #{@interval}")
+        sleep @interval
+      end
     end
 
-    def save
-      statefile.write(YAML.dump(self))
-      log.info("Saved state to #{statefile}")
-    end
+    class CloudwatchIngestInput::State < Hash
+      class LockFailed < RuntimeError; end
+      attr_accessor :statefile
 
-    def close
-      statefile.close
-    end
+      def initialize(filepath)
+        self.statefile = Pathname.new(filepath).open('w')
+        unless statefile.exists?
+          log.warn("No state file #{statefile} Creating a new one.")
+          begin
+            save
+          rescue => boom
+            log.error("Unable to create new state file #{statefile}: #{boom}")
+          end
+        end
+
+        # Attempt to obtain an exclusive flock on the file and raise and
+        # exception if we can't
+        log.info("Obtaining exclusive lock on state file #{statefile}")
+        lockstatus = statefile.flock(File::LOCK_EX | File::LOCK_NB)
+        raise CloudwatchIngestInput::State::LockFailed if lockstatus == false
 
-    def prune(log_groups)
-      groups_before = keys.size
-      delete_if { |k, _v| true unless log_groups.key?(k) }
-      log.info("Pruned #{groups_before - keys.size} keys from state file")
+        merge!(YAML.safe_load(statefile.read))
+        log.info("Loaded state for #{keys.size} log groups from #{statefile}")
+      end
 
-      # TODO: also prune streams as these are most likely to be transient
+      def save
+        statefile.write(YAML.dump(self))
+        log.info("Saved state to #{statefile}")
+      end
+
+      def close
+        statefile.close
+      end
+
+      def prune(log_groups)
+        groups_before = keys.size
+        delete_if { |k, _v| true unless log_groups.key?(k) }
+        log.info("Pruned #{groups_before - keys.size} keys from state file")
+
+        # TODO: also prune streams as these are most likely to be transient
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-ingest
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Sam Pointer