fluent-plugin-cloudwatch-ingest 0.5.4 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2ae5771e3b9826c304a17a701823a7f04d756cfe
-  data.tar.gz: 73fbfc8239d034a6b48de309347f7104792587ac
+  metadata.gz: 97564b12944b487966ac98b5f0175b1fed34b2d9
+  data.tar.gz: 7bde517b6c51e6b78902da36d090d3ddb7594e4e
 SHA512:
-  metadata.gz: 028695acdce8b76c387b6acbace10522b508a025289a4ed18b8cbf85ecb5e3dd5249541d8888ee083ce0f9c907bebaeaa9808274fe504b9d96d113955a9bdc25
-  data.tar.gz: c62d7a7ceae54c17183fc77e85963ef851769094fb91f33ebf5af6631133b604091c186b3aca0970ca1c8b909c21ce395c51cf5d21992d61c6516856f97e178c
+  metadata.gz: 1d5ee8f52a6c7669ab9386b9874f3f227d20f5983550d65bcfd389925b6af3b84bcfa76f59f788547902930fad9ddcfabb60c1b0ff92e93d86497ec01050c064
+  data.tar.gz: 86f6914db8742f1c4eb852a7faea833c11714563ffee10c7f0c3a37acf83d5f713e5baaa2717ac4811f72ae1cba99cea2515f28782c3ad6ab8b189eec579a2b0

data/README.md

@@ -39,10 +39,12 @@ Or install it yourself as:
   limit_events 10000
   state_file_name /mnt/nfs/cloudwatch.state
   interval 60
-  api_interval 5          # Time to wait between API call failures before retry
-  limit_events 10000      # Number of events to fetch in any given iteration
-  event_start_time 0      # Do not fetch events before this time (UNIX epoch, miliseconds)
-  oldest_logs_first false # When true fetch the oldest logs first
+  api_interval 5            # Time to wait between API call failures before retry
+  limit_events 10000        # Number of events to fetch in any given iteration
+  event_start_time 0        # Do not fetch events before this time (UNIX epoch, miliseconds)
+  oldest_logs_first false   # When true fetch the oldest logs first
+  telemetry false           # Produce statsd telemetry
+  statsd_endpoint localhost # Endpoint to which telemetry should be sent
   <parse>
     @type cloudwatch_ingest
     expression /^(?<message>.+)$/

data/fluent-plugin-cloudwatch-ingest.gemspec

@@ -35,4 +35,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'fluentd', '~>0.14.13'
   spec.add_dependency 'aws-sdk', '~>2.8.4'
   spec.add_dependency 'multi_json', '~>1.12'
+  spec.add_dependency 'statsd-ruby', '~>1.4.0'
 end

data/lib/fluent/plugin/cloudwatch/ingest/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Ingest
-        VERSION = '0.5.4'.freeze
+        VERSION = '0.6.0'.freeze
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_ingest.rb

@@ -24,7 +24,7 @@ module Fluent::Plugin
     desc 'Log stream name or prefix. Not setting means "all"'
     config_param :log_stream_name_prefix, :string, default: ''
     desc 'State file name'
-    config_param :state_file_name, :string, default: '/var/spool/td-agent/cloudwatch.state' # rubocop:disable all
+    config_param :state_file_name, :string, default: '/var/spool/td-agent/cloudwatch.state' # rubocop:disable LineLength
     desc 'Fetch logs every interval'
     config_param :interval, :time, default: 60
     desc 'Time to pause between API call failures and limits'
@@ -39,6 +39,10 @@ module Fluent::Plugin
     config_param :event_start_time, :integer, default: 0
     desc 'Fetch the oldest logs first'
     config_param :oldest_logs_first, :bool, default: false
+    desc 'Turn on telemetry'
+    config_param :telemetry, :bool, default: false
+    desc 'Statsd endpoint to which telemetry should be written'
+    config_param :statsd_endpoint, :string, default: 'localhost'
     config_section :parse do
       config_set_default :@type, 'cloudwatch_ingest'
       desc 'Regular expression with which to parse the event message'
@@ -58,6 +62,15 @@ module Fluent::Plugin
       log.info('Starting fluentd-plugin-cloudwatch-ingest')
     end
 
+    def metric(method, name, value = 0)
+      case method
+      when :increment
+        @statsd.send(method, name) if @telemetry
+      else
+        @statsd.send(method, name, value) if @telemetry
+      end
+    end
+
     def configure(conf)
       super
       compat_parameters_convert(conf, :parser)
@@ -72,6 +85,9 @@ module Fluent::Plugin
         raise Fluent::ConfigError, 'parse/event_time is required.'
       end
 
+      # Configure telemetry, if enabled
+      @statsd = Statsd.new @statsd_endpoint, 8125 if @telemetry
+
       @parser = parser_create(conf: parser_config)
       log.info('Configured fluentd-plugin-cloudwatch-ingest')
     end
@@ -92,7 +108,7 @@ module Fluent::Plugin
           role_session_name: @sts_session_name
         )
 
-        log.info("Using STS for authentication with source account ARN: #{@sts_arn}, session name: #{@sts_session_name}") # rubocop:disable all
+        log.info("Using STS for authentication with source account ARN: #{@sts_arn}, session name: #{@sts_session_name}") # rubocop:disable LineLength
       else
         log.info('Using local instance IAM role for authentication')
       end
@@ -111,6 +127,7 @@ module Fluent::Plugin
     def emit(event, log_group_name, log_stream_name)
       @parser.parse(event, log_group_name, log_stream_name) do |time, record|
         router.emit(@tag, time, record)
+        metric(:increment, 'events.emitted.success')
       end
     end
 
@@ -121,6 +138,7 @@ module Fluent::Plugin
       next_token = nil
       loop do
         begin
+          metric(:increment, 'api.calls.describeloggroups.attempted')
           response = if !log_group_prefix.empty?
                        @aws.describe_log_groups(
                          log_group_name_prefix: log_group_prefix,
@@ -137,6 +155,7 @@ module Fluent::Plugin
           next_token = response.next_token
         rescue => boom
           log.error("Unable to retrieve log groups: #{boom}")
+          metric(:increment, 'api.calls.describeloggroups.failed')
           next_token = nil
           sleep @api_interval
           retry
@@ -152,6 +171,7 @@ module Fluent::Plugin
       next_token = nil
       loop do
         begin
+          metric(:increment, 'api.calls.describelogstreams.attempted')
           response = if !log_stream_name_prefix.empty?
                        @aws.describe_log_streams(
                          log_group_name: log_group_name,
@@ -169,7 +189,8 @@ module Fluent::Plugin
           break unless response.next_token
           next_token = response.next_token
         rescue => boom
-          log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom}") # rubocop:disable all
+          log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom}") # rubocop:disable LineLength
+          metric(:increment, 'api.calls.describelogstreams.failed')
           log_streams = []
           next_token = nil
           sleep @api_interval
@@ -206,6 +227,7 @@ module Fluent::Plugin
             end
 
             begin
+              metric(:increment, 'api.calls.getlogevents.attempted')
               response = @aws.get_log_events(
                 log_group_name: group,
                 log_stream_name: stream,
@@ -220,6 +242,7 @@ module Fluent::Plugin
                   emit(e, group, stream)
                 rescue => boom
                   log.error("Failed to emit event #{e}: #{boom}")
+                  metric(:increment, 'events.emitted.failed')
                 end
               end
 
@@ -227,7 +250,8 @@ module Fluent::Plugin
               # in this iteration, store the forward token
               state.store[group][stream] = response.next_forward_token
             rescue => boom
-              log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom}") # rubocop:disable all
+              log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom}") # rubocop:disable LineLength
+              metric(:increment, 'api.calls.getlogevents.failed')
               sleep @api_interval
               retry
             end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-ingest
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.6.0
 platform: ruby
 authors:
 - Sam Pointer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-16 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: statsd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
 description: Fluentd plugin to ingest AWS Cloudwatch logs
 email:
 - san@outsidethe.net