fluent-plugin-cloudwatch-ingest-chaeyk 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fa4e01f170abf58d4e3b049dac9189d587b2d997
-  data.tar.gz: 5e92f9080af46a5840a73ddb8755409d0be2d6f5
+  metadata.gz: 6e539d42376be0deb2bca31af801a1d18d0c2a54
+  data.tar.gz: 0939b1482b07c776aa3b62228f021e6a2e201bd3
 SHA512:
-  metadata.gz: 61d65b2148af75fa53e6d334c5cac2a9b83af13288c9a7448b4cb83e1465433921f0ad02e89e15e3d568d08728ea3f57454c3aba703f03e5803076ea1197f79c
-  data.tar.gz: 5d562ef8945fad2f3ef030e39ad2b222e1391ce1bca129ce1c31c3e12a3461c9e4d5804ef045d1a6999d4f1ca7d7d4c0ea0398eba25f458fe13b5bdf0aee7ac7
+  metadata.gz: 56593d2fb6a811c6199feb4ad82158a33f28d4503be0b313bbebb2e93be6ef5c33107a17c39c08d5b8d0336376df70db1b566a1cf36a8e0bac75c4e9f66cddd0
+  data.tar.gz: eaebcb0661295d542a1d76ac8991b8c68fd0fbe88486feca4a089d25da21755b067ded2ac36e1e2081b83768ecc479b4b8d716ce32756a5c054c6080a3f35b07

data/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in fluent-plugin-cloudwatch-ingest.gemspec
+# Specify your gem's dependencies in fluent-plugin-cloudwatch-ingest-chaeyk.gemspec
 gemspec

data/circle.yml

@@ -6,6 +6,6 @@ test:
 deployment:
   release:
     tag: /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$/
-    owner: sampointer
+    owner: chaeyk
     commands:
       - bin/deploy

data/{fluent-plugin-cloudwatch-ingest.gemspec → fluent-plugin-cloudwatch-ingest-chaeyk.gemspec}

@@ -6,13 +6,13 @@ require 'fluent/plugin/cloudwatch/ingest/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-cloudwatch-ingest-chaeyk'
-  spec.version       = Fluent::Plugin::Cloudwatch::Ingest::VERSION
-  spec.authors       = ['Sam Pointer']
-  spec.email         = ['san@outsidethe.net']
+  spec.version       = Fluent::Plugin::Cloudwatch::Ingest::Chaeyk::VERSION
+  spec.authors       = ['chaeyk']
+  spec.email         = ['chaeyk@gmail.com']
 
   spec.summary       = 'Fluentd plugin to ingest AWS Cloudwatch logs'
   spec.description   = 'Fluentd plugin to ingest AWS Cloudwatch logs'
-  spec.homepage      = 'https://github.com/sampointer/fluent-plugin-cloudwatch-ingest'
+  spec.homepage      = 'https://github.com/chaeyk/fluent-plugin-cloudwatch-ingest-chaeyk'
 
   # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
   # delete this section to allow pushing this gem to any host.
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.10'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 10.5'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'
 

data/lib/fluent/plugin/cloudwatch/ingest/version.rb

@@ -2,7 +2,9 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Ingest
-        VERSION = '0.3.2'.freeze
+        module Chaeyk
+          VERSION = '0.4.0'.freeze
+        end
       end
     end
   end

data/lib/fluent/plugin/{in_cloudwatch_ingest.rb → in_cloudwatch_ingest_chaeyk.rb}

@@ -8,7 +8,7 @@ require 'psych'
 
 module Fluent::Plugin
   class CloudwatchIngestInput < Fluent::Plugin::Input
-    Fluent::Plugin.register_input('cloudwatch_ingest', self)
+    Fluent::Plugin.register_input('cloudwatch_ingest_chaeyk', self)
     helpers :compat_parameters, :parser
 
     desc 'The region of the source cloudwatch logs'
@@ -37,8 +37,10 @@ module Fluent::Plugin
     config_param :limit_events, :integer, default: 10_000
     desc 'Do not fetch events before this time'
     config_param :event_start_time, :integer, default: 0
+    desc 'Fetch the oldest logs first'
+    config_param :oldest_logs_first, :bool, default: false
     config_section :parse do
-      config_set_default :@type, 'cloudwatch_ingest'
+      config_set_default :@type, 'cloudwatch_ingest_chaeyk'
       desc 'Regular expression with which to parse the event message'
       config_param :expression, :string, default: '^(?<message>.+)$'
       desc 'Take the timestamp from the event rather than the expression'
@@ -134,7 +136,7 @@ module Fluent::Plugin
           break unless response.next_token
           next_token = response.next_token
         rescue => boom
-          log.error("Unable to retrieve log groups: #{boom}")
+          log.error("Unable to retrieve log groups: #{boom.inspect}")
           next_token = nil
           sleep @api_interval
           retry
@@ -167,7 +169,7 @@ module Fluent::Plugin
           break unless response.next_token
           next_token = response.next_token
         rescue => boom
-          log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom}") # rubocop:disable all
+          log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom.inspect}") # rubocop:disable all
           log_streams = []
           next_token = nil
           sleep @api_interval
@@ -184,25 +186,36 @@ module Fluent::Plugin
         begin
           state = State.new(@state_file_name, log)
         rescue => boom
-          log.info("Failed lock state. Sleeping for #{@interval}: #{boom}")
+          log.info("Failed lock state. Sleeping for #{@interval}: #{boom.inspect}")
           sleep @interval
-          retry
+          next
         end
 
+        event_count = 0
+
         # Fetch the streams for each log group
         log_groups(@log_group_name_prefix).each do |group|
           # For each log stream get and emit the events
           log_streams(group, @log_stream_name_prefix).each do |stream|
+            if ! state.store[group][stream]
+              state.store[group][stream] = {}
+            end
+
             # See if we have some stored state for this group and stream.
             # If we have then use the stored forward_token to pick up
             # from that point. Otherwise start from the start.
-            if state.store[group] && state.store[group][stream]
-              stream_token =
-                (state.store[group][stream] if state.store[group][stream])
+            if state.store[group][stream]['token']
+              stream_token = state.store[group][stream]['token']
             else
               stream_token = nil
             end
 
+            if state.store[group][stream]['timestamp']
+              stream_timestamp = state.store[group][stream]['timestamp']
+            else
+              stream_timestamp = @event_start_time
+            end
+
             begin
               response = @aws.get_log_events(
                 log_group_name: group,
@@ -210,24 +223,57 @@ module Fluent::Plugin
                 next_token: stream_token,
                 limit: @limit_events,
                 start_time: @event_start_time,
-                start_from_head: true
+                start_from_head: @oldest_logs_first
               )
 
               response.events.each do |e|
                 begin
                   emit(e, group, stream)
+                  event_count = event_count + 1
                 rescue => boom
-                  log.error("Failed to emit event #{e}: #{boom}")
+                  log.error("Failed to emit event #{e}: #{boom.inspect}")
                 end
               end
 
               # Once all events for this stream have been processed,
               # in this iteration, store the forward token
-              state.store[group][stream] = response.next_forward_token
+              state.store[group][stream]['token'] = response.next_forward_token
+              state.store[group][stream]['timestamp'] = response.events.last ? response.events.last.timestamp : stream_timestamp
+            rescue Aws::CloudWatchLogs::Errors::InvalidParameterException => boom
+              log.error("cloudwatch token is expired or broken. trying with timestamp.");
+
+              # try again with timestamp instead of forward token
+              begin
+                response = @aws.get_log_events(
+                  log_group_name: group,
+                  log_stream_name: stream,
+                  limit: @limit_events,
+                  start_time: stream_timestamp,
+                  start_from_head: true
+                )
+
+                response.events.each do |e|
+                  begin
+                    emit(e, group, stream)
+                    event_count = event_count + 1
+                  rescue => boom
+                    log.error("Failed to emit event #{e}: #{boom.inspect}")
+                  end
+                end
+
+                # Once all events for this stream have been processed,
+                # in this iteration, store the forward token
+                state.store[group][stream]["token"] = response.next_forward_token
+                state.store[group][stream]['timestamp'] = response.events.last ? response.events.last.timestamp : steam_timestamp
+              rescue => boom
+                log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom.inspect}") # rubocop:disable all
+                sleep @api_interval
+                next
+              end
             rescue => boom
-              log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom}") # rubocop:disable all
+              log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom.inspect}") # rubocop:disable all
               sleep @api_interval
-              retry
+              next
             end
           end
         end
@@ -238,10 +284,18 @@ module Fluent::Plugin
           state.save
           state.close
         rescue
-          log.error("Unable to save state file: #{boom}")
+          log.error("Unable to save state file: #{boom.inspect}")
         end
-        log.info("Pausing for #{@interval}")
-        sleep @interval
+
+        if event_count > 0
+          sleep_interval = @interval
+        else
+          sleep_interval = @api_interval # when there is no events, slow down
+        end
+
+        log.info("#{event_count} events processed.")
+        log.info("Pausing for #{sleep_interval}")
+        sleep sleep_interval
       end
     end
 
@@ -252,7 +306,7 @@ module Fluent::Plugin
       def initialize(filepath, log)
         @filepath = filepath
         @log = log
-        @store = Hash.new { |h, k| h[k] = {} }
+        @store = Hash.new { |h, k| h[k] = Hash.new { |h1, k1| h1[k1] = {} } }
 
         if File.exist?(filepath)
           self.statefile = Pathname.new(@filepath).open('r+')
@@ -262,7 +316,7 @@ module Fluent::Plugin
             self.statefile = Pathname.new(@filepath).open('w+')
             save
           rescue => boom
-            @log.error("Unable to create new file #{statefile.path}: #{boom}")
+            @log.error("Unable to create new file #{statefile.path}: #{boom.inspect}")
           end
         end
 
@@ -272,12 +326,26 @@ module Fluent::Plugin
         lockstatus = statefile.flock(File::LOCK_EX | File::LOCK_NB)
         raise CloudwatchIngestInput::State::LockFailed if lockstatus == false
 
-        @store.merge!(Psych.safe_load(statefile.read))
-        @log.info("Loaded #{@store.keys.size} groups from #{statefile.path}")
+        begin
+          @store.merge!(Psych.safe_load(statefile.read))
+
+          # Migrate old state file
+          @store.each { |group, streams|
+            streams.update(streams) { |name, stream|
+              (stream.is_a? String) ? { 'token' => stream, 'timestamp' => Time.now.to_i } : stream
+            }
+          }
+
+          @log.info("Loaded #{@store.keys.size} groups from #{statefile.path}")
+        rescue
+          statefile.close
+          raise
+        end
       end
 
       def save
         statefile.rewind
+        statefile.truncate(0)
         statefile.write(Psych.dump(@store))
         @log.info("Saved state to #{statefile.path}")
         statefile.rewind

data/lib/fluent/plugin/{parser_cloudwatch_ingest.rb → parser_cloudwatch_ingest_chaeyk.rb}

@@ -4,7 +4,7 @@ require 'fluent/time'
 module Fluent
   module Plugin
     class CloudwatchIngestParser < RegexpParser
-      Plugin.register_parser('cloudwatch_ingest', self)
+      Plugin.register_parser('cloudwatch_ingest_chaeyk', self)
 
       config_param :expression, :string, default: '^(?<message>.+)$'
       config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S.%L'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-ingest-chaeyk
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
-- Sam Pointer
+- chaeyk
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-13 00:00:00.000000000 Z
+date: 2017-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '10.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '10.5'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -96,7 +96,7 @@ dependencies:
         version: 2.8.4
 description: Fluentd plugin to ingest AWS Cloudwatch logs
 email:
-- san@outsidethe.net
+- chaeyk@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -114,12 +114,12 @@ files:
 - bin/deploy
 - bin/setup
 - circle.yml
-- fluent-plugin-cloudwatch-ingest.gemspec
+- fluent-plugin-cloudwatch-ingest-chaeyk.gemspec
 - lib/fluent/plugin/cloudwatch/ingest.rb
 - lib/fluent/plugin/cloudwatch/ingest/version.rb
-- lib/fluent/plugin/in_cloudwatch_ingest.rb
-- lib/fluent/plugin/parser_cloudwatch_ingest.rb
-homepage: https://github.com/sampointer/fluent-plugin-cloudwatch-ingest
+- lib/fluent/plugin/in_cloudwatch_ingest_chaeyk.rb
+- lib/fluent/plugin/parser_cloudwatch_ingest_chaeyk.rb
+homepage: https://github.com/chaeyk/fluent-plugin-cloudwatch-ingest-chaeyk
 licenses: []
 metadata:
   allowed_push_host: https://rubygems.org