fluent-plugin-throttle 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 65759b9fff6dd920e4d1963a9d7ef7d786cd4708
-  data.tar.gz: fce8128e0943af6ce0dcb412658dcebdb426ea00
+  metadata.gz: 96baca37d87b547aadf4ef128d973f5605477242
+  data.tar.gz: bb0ec1c1d81b4db2048dafac99d7b795caf2fc68
 SHA512:
-  metadata.gz: 32ac8f613190595a760cb2bdc44c7e1e270824c86ebcff96179751df63325798a2384eb8f4f4557455db6020698acf950de5f51b375be802177a504e6da671c1
-  data.tar.gz: fc74e2b98c362d0428ae9b24464a5466bec1ec17cef2827bf206ce707450f8700ffbb70453fc8b8c45338ded45879299695ec795404a4d6a9405000f2662299b
+  metadata.gz: a85cdcbdae70af12c549012a58a2e640727c8fb9dbb12e0753499ee0091a124bd8ac90097bf89193293849de9aea6cdab4e0e3b986856aab6396f18759dfe88b
+  data.tar.gz: 21482c46242a78e2d7d2150cead76734afdaf937be5c2d714c1d2c6ac8afa79472a89ced7cffcea7a44ab0e6807c6d17bd9b4b5c5dab6adc54a851243a2dfacd

data/README.md

@@ -26,7 +26,7 @@ $ gem install fluent-plugin-throttle
 
 ## Configuration
 
-### group\_key
+#### group\_key
 
 Default: `kubernetes.container_name`.
 
@@ -40,13 +40,13 @@ the group key resolve to "random":
 
 If the group cannot be resolved, am anonymous group is used.
 
-## group\_bucket\_period\_s
+#### group\_bucket\_period\_s
 
 Default: `60` (60 second).
 
 This is the period of of time over which `group_bucket_limit` applies.
 
-## group\_bucket\_limit
+#### group\_bucket\_limit
 
 Default: `6000` (logs per `group_bucket_period_s`).
 
@@ -69,10 +69,10 @@ logs per minute. However `60/60s` will readily emit 60 logs within the first
 second then nothing for the remaining 59 seconds. While the `1/1s` will only
 emit the first log of every second.
 
-## group\_reset\_rate\_s
+#### group\_reset\_rate\_s
 
 Default: `group_bucket_limit/group_bucket_period_s` (logs per `group_bucket_period_s`).
-Max: `group_bucket_limit`.
+Maximum: `group_bucket_limit/group_bucket_period_s`.
 
 After a group has exceeded its bucket limit, logs are dropped until the rate
 per second falls below or equal to `group_reset_rate_s`.
@@ -105,12 +105,13 @@ after a single breach of the limit until the next restart of fluentd.
 
 A value of `-1` disables the feature.
 
-## group\_warning\_hz
+#### group\_warning\_delay\_s
 
-Default: `0.1` (10 seconds).
+Default: `10` (seconds).
 
 When a group reaches its limit and as long as it is not reset, a warning
-message with the current log rate for the group is logged at this frequency.
+message with the current log rate of the group is emitted repeatedly. This is
+the delay between every repetition.
 
 ## Copyright
 

data/fluent-plugin-throttle.gemspec

@@ -4,11 +4,11 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-throttle"
-  spec.version       = "0.0.2"
+  spec.version       = "0.0.3"
   spec.authors       = ["François-Xavier Bourlet"]
   spec.email         = ["fx.bourlet@rubrik.com"]
   spec.summary       = %q{Fluentd filter for throttling logs based on a configurable key.}
-  spec.homepage      = "https://github.com/bombela/fluent-plugin-throttle"
+  spec.homepage      = "https://github.com/rubrikinc/fluent-plugin-throttle"
   spec.license       = "Apache-2.0"
 
   spec.files         = `git ls-files`.split($/)

data/lib/fluent/plugin/filter_throttle.rb

@@ -8,9 +8,8 @@ module Fluent
     config_param :group_bucket_period_s, :integer, :default => 60
     config_param :group_bucket_limit, :integer, :default => 6000
     config_param :group_reset_rate_s, :integer, :default => nil
-    config_param :warning_hz, :float, :default => 0.1
+    config_param :group_warning_delay_s, :integer, :default => 10
 
-    Bucket = Struct.new(:emitted, :last_reset)
     Group = Struct.new(
       :rate_count,
       :rate_last_reset,
@@ -40,7 +39,8 @@ module Fluent
       raise "group_reset_rate_s must be <= group_bucket_limit / group_bucket_period_s" \
         unless @group_reset_rate_s <= @group_rate_limit
 
-      @warning_delay = (1.0 / @warning_hz)
+      raise "group_warning_delay_s must be >= 1" \
+        unless @group_warning_delay_s >= 1
     end
 
     def start
@@ -73,35 +73,34 @@ module Fluent
 
       if (now.to_i / @group_bucket_period_s) \
           > (counter.bucket_last_reset.to_i / @group_bucket_period_s)
-        # next time period reached, reset limit.
+        # next time period reached.
 
+        # wait until rate drops back down (if enabled).
         if counter.bucket_count == -1 and @group_reset_rate_s != -1
-          # wait until rate drops back down if needed.
           if counter.aprox_rate < @group_reset_rate_s
             log_rate_back_down(now, group, counter)
           else
-            since_last_warning = now - counter.last_warning
-            if since_last_warning >= @warning_delay
-              log_rate_limit_exceeded(now, group, counter)
-              counter.last_warning = now
-            end
+            log_rate_limit_exceeded(now, group, counter)
             return nil
           end
         end
 
+        # reset counter for the rest of time period.
         counter.bucket_count = 0
         counter.bucket_last_reset = now
-      end
-
-      if counter.bucket_count == -1
-        return nil
+      else
+        # if current time period credit is exhausted, drop the record.
+        if counter.bucket_count == -1
+          log_rate_limit_exceeded(now, group, counter)
+          return nil
+        end
       end
 
       counter.bucket_count += 1
 
+      # if we are out of credit, we drop logs for the rest of the time period.
       if counter.bucket_count > @group_bucket_limit
         log_rate_limit_exceeded(now, group, counter)
-        counter.last_warning = now
         counter.bucket_count = -1
         return nil
       end
@@ -114,7 +113,12 @@ module Fluent
     end
 
     def log_rate_limit_exceeded(now, group, counter)
-      $log.warn("rate exceeded", log_items(now, group, counter))
+      emit = counter.last_warning == nil ? true \
+        : (now - counter.last_warning) >= @group_warning_delay_s
+      if emit
+        $log.warn("rate exceeded", log_items(now, group, counter))
+        counter.last_warning = now
+      end
     end
 
     def log_rate_back_down(now, group, counter)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-throttle
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - François-Xavier Bourlet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-20 00:00:00.000000000 Z
+date: 2018-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -105,7 +105,7 @@ files:
 - README.md
 - fluent-plugin-throttle.gemspec
 - lib/fluent/plugin/filter_throttle.rb
-homepage: https://github.com/bombela/fluent-plugin-throttle
+homepage: https://github.com/rubrikinc/fluent-plugin-throttle
 licenses:
 - Apache-2.0
 metadata: {}