fluent-plugin-cloudwatch-ingest 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e71179fd143c24d2f1e0c41e5368b93f69ab9ee2
-  data.tar.gz: 7f5f859479b39059a3fb5086fdfa1cb79193d828
+  metadata.gz: 852a0773b4fd72b1ca24faebd4624b5732d3d916
+  data.tar.gz: 9815dd011589740f0af019e450ff1def0682f11a
 SHA512:
-  metadata.gz: 8f036ac6cc92eaa5762986352b03a339a498fe8de4690e1b48572d7a0232ef2975f75eb34b6a605b7e9fed510dae0225d7454ca228c4e0e20e8faec2d2538a36
-  data.tar.gz: 74823d37c5ee4a19d71d40bcfe496a3a399e01b700bc9129c5c83512e3529f3a643da6ff4e037d08c310641aba0c71d30d299af0892889903ed2c77353be807c
+  metadata.gz: 10ab86bf0fde33eb65869c2db408a3a9bc1658807d4d3e6e8e5b7bd35e9f68f81c4e5a2270dc636e7dbadb3bd306dd9f39c683eba7eb05d9b2b8149df3415845
+  data.tar.gz: 2998fe65f16a65b8582fb6b2ec0ca4b0e3ad20262e229d8f0140cd5bfb7d68bbd93b4c72ad00712ffc8f075db7cae32bc968e82dac2d910fee9f72375015574d

data/CHANGELOG.md

@@ -47,3 +47,13 @@
 * Add telemetry to the parser
 
 Both of these changes are designed to make debugging ingestion problems from high-volume log groups easier.
+
+## 1.3.0
+
+* Add `log_group_exclude_regexp` to allow optional exclusion of log groups by regexp
+
+## 1.4.0
+
+* Refuse to emit records with a blank (or newline only) message
+* Emit metric `events.emitted.blocked` to expose these alongside logging
+* Add plugin skew time to telemetry optionally emitted from the parser

data/README.md

@@ -44,6 +44,7 @@ Or install it yourself as:
   limit_events 10000        # Number of events to fetch in any given iteration
   event_start_time 0        # Do not fetch events before this time (UNIX epoch, miliseconds)
   oldest_logs_first false   # When true fetch the oldest logs first
+  drop_blank_events true    # Fluentd may throw an exception if a blank event is emitted
   telemetry false           # Produce statsd telemetry
   statsd_endpoint localhost # Endpoint to which telemetry should be sent
   <parse>
@@ -106,6 +107,7 @@ api.calls.getlogevents.attempted
 api.calls.getlogevents.failed
 api.calls.getlogevents.invalid_token
 events.emitted.success
+events.emitted.blocked
 ```
 
 Likewise when telemetry is enabled for the parser, the emitted metrics are:
@@ -116,6 +118,7 @@ parser.record.success
 parser.json.success     # if json parsing is enabled
 parser.json.failed      # if json parsing is enabled
 parser.ingestion_skew   # the difference between `timestamp` and `ingestion_time` as returned by the Cloudwatch API
+parser.plugin_skew      # the difference between "now" and `timestamp`
 ```
 
 ### Sub-second timestamps

data/lib/fluent/plugin/cloudwatch/ingest/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Ingest
-        VERSION = '1.3.0'.freeze
+        VERSION = '1.4.0'.freeze
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_ingest.rb

@@ -43,6 +43,8 @@ module Fluent::Plugin
     config_param :event_start_time, :integer, default: 0
     desc 'Fetch the oldest logs first'
     config_param :oldest_logs_first, :bool, default: false
+    desc 'Refuse to emit events with a blank "message" field'
+    config_param :drop_blank_events, :bool, default: true
     desc 'Turn on telemetry'
     config_param :telemetry, :bool, default: false
     desc 'Statsd endpoint to which telemetry should be written'
@@ -136,6 +138,11 @@ module Fluent::Plugin
 
     def emit(event, log_group_name, log_stream_name)
       @parser.parse(event, log_group_name, log_stream_name) do |time, record|
+        if record['message'].chomp.empty? && @drop_blank_events
+          log.warn("Event is blank or contains only a newline, refusing to emit. group: #{log_group_name}, stream: #{log_stream_name}, event: #{event}") # rubocop:disable LineLength
+          metric(:increment, 'events.emitted.blocked')
+          next
+        end
         router.emit(@tag, time, record)
         metric(:increment, 'events.emitted.success')
       end

data/lib/fluent/plugin/parser_cloudwatch_ingest.rb

@@ -94,6 +94,15 @@ module Fluent
           )
         end
 
+        # Optionally emit @timestamp and plugin ingestion time skew
+        if @telemetry
+          metric(
+            :gauge,
+            'parser.plugin_skew',
+            now.strftime('%Q').to_i - event.timestamp
+          )
+        end
+
         # We do String processing on the event time here to
         # avoid rounding errors introduced by floating point
         # arithmetic.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-ingest
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Sam Pointer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-20 00:00:00.000000000 Z
+date: 2017-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler