fluent-plugin-throttle 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b2e1a36b7725f3104af98d1fb78418cda39bbd485a4c934652a9e01a179bb23
-  data.tar.gz: 55a63790bcf7b60f27fdf6a886791feb57a7f8245c7607a1c182890ea95122e6
+  metadata.gz: 42672474afdd0860dfd06b5dfa7547c1e3b6365f1a995bd288b5928d54afdcaa
+  data.tar.gz: 78cb604160da44a19eee7473e56590af7d7076042b143bc3138b8198b35c8f97
 SHA512:
-  metadata.gz: 0ed4d59ec171b432ceb4d0f7bfd95842dbba2deef99d8fa81c12aa6531b3a72a87668f26df08a75189d203c5b9e31192d6cc02b3c60f70fbb7865a311ea35bc2
-  data.tar.gz: f6fddde539b046be640c99327276e3f965c605f87a1949e25e29e2339d109b29cc7f874608bdd164bc6a1c7a6b4d34f7f0441e83432f4a6a41ee77e7270b1a95
+  metadata.gz: 6fe4f1c8d5c250ab8e4945086460227040ed87357ad8565ba484710a405d7f1a14461c1db2bd16f921945916e69e035c2d92c01b5fe91e57eacb9644a4d05349
+  data.tar.gz: 10d3b4572029fc79cbfb09cf223e82d2af69f3c1239165a6690d97a1ee512a27f27b05e945db195ca4c8f924b2e0aa2688f4e670364145f789f9f8aeefebf3dc

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-throttle (0.0.3)
+    fluent-plugin-throttle (0.0.5)
       fluentd (~> 1.1)
 
 GEM
@@ -15,19 +15,18 @@ GEM
       thor (>= 0.14.0)
     byebug (10.0.2)
     coderay (1.1.2)
-    cool.io (1.5.3)
+    concurrent-ruby (1.1.6)
+    cool.io (1.6.0)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    dig_rb (1.0.1)
-    fluentd (1.2.3)
+    fluentd (1.11.1)
       cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
+      msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.0.4, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 1.0)
+      tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
     hashdiff (0.3.7)
@@ -39,7 +38,7 @@ GEM
     minitest (5.11.3)
     mocha (1.6.0)
       metaclass (~> 0.0.1)
-    msgpack (1.2.4)
+    msgpack (1.3.3)
     power_assert (1.1.3)
     pry (0.11.3)
       coderay (~> 1.1.0)
@@ -50,18 +49,17 @@ GEM
     public_suffix (3.0.2)
     rake (12.3.1)
     safe_yaml (1.0.4)
-    serverengine (2.0.7)
+    serverengine (2.2.1)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     single_cov (1.1.0)
-    strptime (0.2.3)
+    strptime (0.2.4)
     test-unit (3.2.8)
       power_assert
     thor (0.20.0)
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2018.5)
+    tzinfo (2.0.2)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2020.1)
       tzinfo (>= 1.0.0)
     webmock (3.4.2)
       addressable (>= 2.3.6)
@@ -86,4 +84,4 @@ DEPENDENCIES
   webmock (~> 3.3)
 
 BUNDLED WITH
-   1.16.2
+   1.17.2

data/README.md

@@ -2,7 +2,7 @@
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/rubrikinc/fluent-plugin-throttle/blob/master/LICENSE)
 
-A sentry pluging to throttle logs. Logs are grouped by a configurable key. When
+A sentry plugin to throttle logs. Logs are grouped by a configurable key. When
 a group exceeds a configuration rate, logs are dropped for this group.
 
 ## Installation

data/fluent-plugin-throttle.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-throttle"
-  spec.version       = "0.0.4"
+  spec.version       = "0.0.5"
   spec.authors       = ["François-Xavier Bourlet"]
   spec.email         = ["fx.bourlet@rubrik.com"]
   spec.summary       = %q{Fluentd filter for throttling logs based on a configurable key.}

data/lib/fluent/plugin/filter_throttle.rb

@@ -51,6 +51,8 @@ module Fluent::Plugin
       raise "group_bucket_period_s must be > 0" \
         unless @group_bucket_period_s > 0
 
+      @group_gc_timeout_s = 2 * @group_bucket_period_s
+
       raise "group_bucket_limit must be > 0" \
         unless @group_bucket_limit > 0
 
@@ -83,17 +85,26 @@ module Fluent::Plugin
       now = Time.now
       rate_limit_exceeded = @group_drop_logs ? nil : record # return nil on rate_limit_exceeded to drop the record
       group = extract_group(record)
-      counter = (@counters[group] ||= Group.new(0, now, 0, 0, now, nil))
-      counter.rate_count += 1
+      
+      # Ruby hashes are ordered by insertion. 
+      # Deleting and inserting moves the item to the end of the hash (most recently used)
+      counter = @counters[group] = @counters.delete(group) || Group.new(0, now, 0, 0, now, nil)
 
+      counter.rate_count += 1
       since_last_rate_reset = now - counter.rate_last_reset
       if since_last_rate_reset >= 1
-        # compute and store rate/s at most every seconds.
+        # compute and store rate/s at most every second
         counter.aprox_rate = (counter.rate_count / since_last_rate_reset).round()
         counter.rate_count = 0
         counter.rate_last_reset = now
       end
 
+      # try to evict the least recently used group
+      lru_group, lru_counter = @counters.first
+      if !lru_group.nil? && now - lru_counter.rate_last_reset > @group_gc_timeout_s
+        @counters.delete(lru_group)
+      end
+
       if (now.to_i / @group_bucket_period_s) \
           > (counter.bucket_last_reset.to_i / @group_bucket_period_s)
         # next time period reached.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-throttle
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - François-Xavier Bourlet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-28 00:00:00.000000000 Z
+date: 2020-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler