fluent-plugin-cloudwatch-ingest-chaeyk 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 791d8ae3a09f23b29a3a0e9a0714e75556339f89
-  data.tar.gz: 4837ba8d518e6f564b9a069e247fe21cc230ec8d
+  metadata.gz: b77c18e98f2c2b14518b200ae364ade70b76d8a9
+  data.tar.gz: 3eb03829d3d30b8a4427f15b96362391e401824a
 SHA512:
-  metadata.gz: df7af18b9f6fd76e2161875092fb7ed88be58a65bc66a23aa2df34f2ce7ac979c40b93a61d850b4b5eb2d9228cdb03cb30c597660175da52d22d284d3dc02d7a
-  data.tar.gz: 04777edff81450b15c3fb6eee7336af2dc5ce423137e7ff2db9d44030c0ac03dd7de34f686cc98ae7c138c2496bf5e7651f964acf54f4644259454c1828f4673
+  metadata.gz: 30c46498a01f39697ed3d24987fdd8fa1d90af203df91eaffccdb1154ce4758cea2d35cd84032d0f4a0ecc0f3c365ebddc5f76a2504247b3eab1d4864380782b
+  data.tar.gz: d0e0a05b240b24324875d29fd6dc8425ca7478aa26b9b879aec1840633315d45331cb398bf999cf3478deeff558a44549a2681b6f53a8403d60ac100688c74b6

data/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in fluent-plugin-cloudwatch-ingest-chaeyk.gemspec
+# Specify your dependencies in fluent-plugin-cloudwatch-ingest-chaeyk.gemspec
 gemspec

data/fluent-plugin-cloudwatch-ingest-chaeyk.gemspec

@@ -6,7 +6,7 @@ require 'fluent/plugin/cloudwatch/ingest/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-cloudwatch-ingest-chaeyk'
-  spec.version       = Fluent::Plugin::Cloudwatch::Ingest::Chaeyk::VERSION
+  spec.version       = Fluent::Plugin::Cloudwatch::Ingest::VERSION
   spec.authors       = ['chaeyk']
   spec.email         = ['chaeyk@gmail.com']
 

data/lib/fluent/plugin/cloudwatch/ingest/version.rb

@@ -2,9 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Ingest
-        module Chaeyk
-          VERSION = '0.4.2'.freeze
-        end
+        VERSION = '0.4.3'.freeze
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_ingest_chaeyk.rb

@@ -181,12 +181,55 @@ module Fluent::Plugin
       return log_streams
     end
 
+    def process_stream(group, stream, next_token, start_time, state)
+      event_count = 0
+
+      response = @aws.get_log_events(
+        log_group_name: group,
+        log_stream_name: stream,
+        next_token: next_token,
+        limit: @limit_events,
+        start_time: start_time,
+        start_from_head: @oldest_logs_first
+      )
+
+      response.events.each do |e|
+        begin
+          emit(e, group, stream)
+          event_count += 1
+        rescue => boom
+          log.error("Failed to emit event #{e}: #{boom.inspect}")
+        end
+      end
+
+      has_stream_timestamp = true if state.store[group][stream]['timestamp']
+
+      if !has_stream_timestamp && response.events.count.zero?
+        # This stream has returned no data ever.
+        # In this case, don't save state (token could be an invalid one)
+      else
+        # Once all events for this stream have been processed,
+        # in this iteration, store the forward token
+        state.new_store[group][stream]['token'] = response.next_forward_token
+        if response.events.last
+          state.new_store[group][stream]['timestamp'] =
+            response.events.last.timestamp
+        else
+          state.new_store[group][stream]['timestamp'] =
+            state.store[group][stream]['timestamp']
+        end
+      end
+
+      return event_count
+    end
+
     def run
       until @finished
         begin
           state = State.new(@state_file_name, log)
         rescue => boom
-          log.info("Failed lock state. Sleeping for #{@interval}: #{boom.inspect}")
+          log.info("Failed lock state. Sleeping for #{@interval}: "\
+                   "#{boom.inspect}")
           sleep @interval
           next
         end
@@ -197,96 +240,45 @@ module Fluent::Plugin
         log_groups(@log_group_name_prefix).each do |group|
           # For each log stream get and emit the events
           log_streams(group, @log_stream_name_prefix).each do |stream|
-            if ! state.store[group][stream]
-              state.store[group][stream] = {}
-            end
+            state.store[group][stream] = {} unless state.store[group][stream]
+
+            log.info("processing stream: #{stream}")
 
             # See if we have some stored state for this group and stream.
             # If we have then use the stored forward_token to pick up
             # from that point. Otherwise start from the start.
-            if state.store[group][stream]['token']
-              stream_token = state.store[group][stream]['token']
-            else
-              stream_token = nil
-            end
-
-            if state.store[group][stream]['timestamp']
-              stream_timestamp = state.store[group][stream]['timestamp']
-            else
-              stream_timestamp = @event_start_time
-            end
 
             begin
-              response = @aws.get_log_events(
-                log_group_name: group,
-                log_stream_name: stream,
-                next_token: stream_token,
-                limit: @limit_events,
-                start_time: @event_start_time,
-                start_from_head: @oldest_logs_first
-              )
-
-              response.events.each do |e|
-                begin
-                  emit(e, group, stream)
-                  event_count = event_count + 1
-                rescue => boom
-                  log.error("Failed to emit event #{e}: #{boom.inspect}")
-                end
-              end
-
-              # Once all events for this stream have been processed,
-              # in this iteration, store the forward token
-              log.info("stream: #{stream} token: #{stream_token} count: #{event_count}")
-              if stream_token or response.events.count > 0
-                log.info("saving: #{response.events}")
-                state.store[group][stream]['token'] = response.next_forward_token
-                state.store[group][stream]['timestamp'] = response.events.last ? response.events.last.timestamp : stream_timestamp
-              else
-                log.info("throw out")
-                state.store[group].delete(stream)
-              end
-            rescue Aws::CloudWatchLogs::Errors::InvalidParameterException => boom
-              log.error("cloudwatch token is expired or broken. trying with timestamp.");
+              event_count += process_stream(group, stream,
+                                            state.store[group][stream]['token'],
+                                            @event_start_time, state)
+            rescue Aws::CloudWatchLogs::Errors::InvalidParameterException
+              log.error('cloudwatch token is expired or broken. '\
+                        'trying with timestamp.')
 
               # try again with timestamp instead of forward token
               begin
-                response = @aws.get_log_events(
-                  log_group_name: group,
-                  log_stream_name: stream,
-                  limit: @limit_events,
-                  start_time: stream_timestamp,
-                  start_from_head: true
-                )
-
-                response.events.each do |e|
-                  begin
-                    emit(e, group, stream)
-                    event_count = event_count + 1
-                  rescue => boom
-                    log.error("Failed to emit event #{e}: #{boom.inspect}")
-                  end
-                end
-
-                # Once all events for this stream have been processed,
-                # in this iteration, store the forward token
-                state.store[group][stream]["token"] = response.next_forward_token
-                state.store[group][stream]['timestamp'] = response.events.last ? response.events.last.timestamp : steam_timestamp
+                timestamp = state.store[group][stream]['timestamp']
+                timestamp = @event_start_time unless timestamp
+
+                event_count += process_stream(group, stream,
+                                              nil, timestamp, state)
               rescue => boom
-                log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom.inspect}") # rubocop:disable all
+                log.error("Unable to retrieve events for stream #{stream} "\
+                          "in group #{group}: #{boom.inspect}") # rubocop:disable all
                 sleep @api_interval
                 next
               end
             rescue => boom
-              log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom.inspect}") # rubocop:disable all
+              log.error("Unable to retrieve events for stream #{stream} "\
+                        "in group #{group}: #{boom.inspect}") # rubocop:disable all
               sleep @api_interval
               next
             end
           end
         end
 
-        log.info('Pruning and saving state')
-        state.prune(log_groups(@log_group_name_prefix)) # Remove dead streams
+        log.info('Saving state')
         begin
           state.save
           state.close
@@ -308,12 +300,13 @@ module Fluent::Plugin
 
     class CloudwatchIngestInput::State
       class LockFailed < RuntimeError; end
-      attr_accessor :statefile, :store
+      attr_accessor :statefile, :store, :new_store
 
       def initialize(filepath, log)
         @filepath = filepath
         @log = log
-        @store = Hash.new { |h, k| h[k] = Hash.new { |h1, k1| h1[k1] = {} } }
+        @store = Hash.new { |h, k| h[k] = Hash.new { |x, y| x[y] = {} } }
+        @new_store = Hash.new { |h, k| h[k] = Hash.new { |x, y| x[y] = {} } }
 
         if File.exist?(filepath)
           self.statefile = Pathname.new(@filepath).open('r+')
@@ -323,7 +316,8 @@ module Fluent::Plugin
             self.statefile = Pathname.new(@filepath).open('w+')
             save
           rescue => boom
-            @log.error("Unable to create new file #{statefile.path}: #{boom.inspect}")
+            @log.error("Unable to create new file #{statefile.path}: "\
+                       "#{boom.inspect}")
           end
         end
 
@@ -337,11 +331,14 @@ module Fluent::Plugin
           @store.merge!(Psych.safe_load(statefile.read))
 
           # Migrate old state file
-          @store.each { |group, streams|
-            streams.update(streams) { |name, stream|
-              (stream.is_a? String) ? { 'token' => stream, 'timestamp' => Time.now.to_i } : stream
-            }
-          }
+          @store.each do |_group, streams|
+            streams.update(streams) do |_name, stream|
+              if stream.is_a? String
+                return { 'token' => stream, 'timestamp' => Time.now.to_i }
+              end
+              return stream
+            end
+          end
 
           @log.info("Loaded #{@store.keys.size} groups from #{statefile.path}")
         rescue
@@ -353,7 +350,7 @@ module Fluent::Plugin
       def save
         statefile.rewind
         statefile.truncate(0)
-        statefile.write(Psych.dump(@store))
+        statefile.write(Psych.dump(@new_store))
         @log.info("Saved state to #{statefile.path}")
         statefile.rewind
       end
@@ -361,14 +358,6 @@ module Fluent::Plugin
       def close
         statefile.close
       end
-
-      def prune(log_groups)
-        groups_before = @store.keys.size
-        @store.delete_if { |k, _v| true unless log_groups.include?(k) }
-        @log.info("Pruned #{groups_before - @store.keys.size} keys from store")
-
-        # TODO: also prune streams as these are most likely to be transient
-      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-ingest-chaeyk
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - chaeyk
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-30 00:00:00.000000000 Z
+date: 2017-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler