fluent-plugin-cloudwatch-ingest 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9099fc9a7106d463763eedd19d543e9e1023231c
-  data.tar.gz: 2d0203a3edd8f96f7a0f25a71dadf79b1d6f8c3d
+  metadata.gz: e71179fd143c24d2f1e0c41e5368b93f69ab9ee2
+  data.tar.gz: 7f5f859479b39059a3fb5086fdfa1cb79193d828
 SHA512:
-  metadata.gz: 41d7a5122379a9e2c8ff975e61b49475e2a1ee31b7ca3bba5aa468e5913f7d1ed2f71e7175f7fb2a2fef3c5a1f844150fc9a35054e0d235152b1fe39ba18132e
-  data.tar.gz: 39c4c78a663a47054c147e785075de17eb70f12933eb42a1f4e821485cb7e59bed17d260babe967b22edb84687213dafdbd6f7fa0a478e61eccb13eff265cc9a
+  metadata.gz: 8f036ac6cc92eaa5762986352b03a339a498fe8de4690e1b48572d7a0232ef2975f75eb34b6a605b7e9fed510dae0225d7454ca228c4e0e20e8faec2d2538a36
+  data.tar.gz: 74823d37c5ee4a19d71d40bcfe496a3a399e01b700bc9129c5c83512e3529f3a643da6ff4e037d08c310641aba0c71d30d299af0892889903ed2c77353be807c

data/README.md

@@ -37,6 +37,7 @@ Or install it yourself as:
   aws_logging_enabled true
   log_group_name_prefix /aws/lambda
   log_stream_name_prefix 2017
+  log_group_exclude_regexp [^A-Z]{1}.* # exclude log groups that start with a captial
   state_file_name /mnt/nfs/cloudwatch.state
   interval 60
   error_interval 5          # Time to wait between error conditions before retry
@@ -52,8 +53,8 @@ Or install it yourself as:
     event_time true                             # take time from the Cloudwatch event, rather than parse it from the body
     inject_group_name true                      # inject the group name into the record
     inject_stream_name true                     # inject the stream name into the record
-    inject_cloudwatch_ingestion_time field_name # inject the `ingestion_time` as returned by the Cloudwatch API
-    inject_plugin_ingestion_time field_name     # inject the 13 digit epoch time at which the plugin ingested the event
+    inject_cloudwatch_ingestion_time field_name # inject the iso8601 `ingestion_time` as returned by the Cloudwatch API
+    inject_plugin_ingestion_time field_name     # inject the iso8601 time at which the plugin ingested the event
     parse_json_body false                       # Attempt to parse the body as json and add structured fields from the result
     fail_on_unparsable_json false               # If the body cannot be parsed as json do not ingest the record
     telemetry false                             # Produce statsd telemetry
@@ -100,6 +101,7 @@ api.calls.describeloggroups.attempted
 api.calls.describeloggroups.failed
 api.calls.describelogstreams.attempted
 api.calls.describelogstreams.failed
+api.calls.describeloggroups.excluded # due to log_group_exclude_regexp
 api.calls.getlogevents.attempted
 api.calls.getlogevents.failed
 api.calls.getlogevents.invalid_token

data/lib/fluent/plugin/cloudwatch/ingest/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Ingest
-        VERSION = '1.2.0'.freeze
+        VERSION = '1.3.0'.freeze
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_ingest.rb

@@ -24,6 +24,8 @@ module Fluent::Plugin
     config_param :log_group_name_prefix, :string, default: ''
     desc 'Log stream name or prefix. Not setting means "all"'
     config_param :log_stream_name_prefix, :string, default: ''
+    desc 'Log group regexp to exclude, despite matching'
+    config_param :log_group_exclude_regexp, :string, default: ''
     desc 'State file name'
     config_param :state_file_name, :string, default: '/var/spool/td-agent/cloudwatch.state' # rubocop:disable LineLength
     desc 'Fetch logs every interval'
@@ -158,7 +160,19 @@ module Fluent::Plugin
                        )
                      end
 
-          response.log_groups.each { |g| log_groups << g.log_group_name }
+          regex = Regexp.new(@log_group_exclude_regexp)
+          response.log_groups.each do |group|
+            if !@log_group_exclude_regexp.empty?
+              if regex.match(group.log_group_name)
+                log.info("Excluding log_group #{group.log_group_name} due to log_group_exclude_regexp #{@log_group_exclude_regexp}") # rubocop:disable LineLength
+                metric(:increment, 'api.calls.describeloggroups.excluded')
+              else
+                log_groups << group.log_group_name
+              end
+            else
+              log_groups << group.log_group_name
+            end
+          end
           break unless response.next_token
           next_token = response.next_token
         rescue => boom

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-ingest
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Sam Pointer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-18 00:00:00.000000000 Z
+date: 2017-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler