fluent-plugin-quota-throttle 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2426b566cfd2e8daca3b7f8906ec9158f0155dc70ce3cf0d6411d28393213ae
-  data.tar.gz: 4e81bc0f686d1ea4c55205abbbc7166d523812f644a17c381ed3265c5a9e1349
+  metadata.gz: 4616f13cc061297e5a1a26dd282723b8696abf657b62b93022aa80a78cbaf1d8
+  data.tar.gz: ba6b6e9f85a436b611dcc6c89693e7f24b8901a7242416136698076c4e4c0e0a
 SHA512:
-  metadata.gz: 8e075b4fea32c31c4c34a1f67b96c60ded654fa6c835d5a3cce55c95d4ffda7923ec2badfc9819f754fa042614f8e783b981ee6d327f10235b409db205148d5d
-  data.tar.gz: 504c88b37e54ab1184f20bbf1096e3ef57a5f2613028e1dffbbb5eec336f37b55b22640bb969e954493e45b7fe5e14144665f44f91c0f0914374ec8f6662e8ff
+  metadata.gz: 8737089035454b8fb25808ed25092f989bea67f8e39998d3f2bae87c46be5f9cbc8006b448f55b327648d30dc8f06a6853092d444478a583dc2a96ffdcaaf81b
+  data.tar.gz: 47dcabe4de812fd2738ba6640b0ace2a870b9a3a9376f2a0256a68773baec1445dae760c28caee653f74032212d89be0b6cceeef93d80064a6d3cf8039b20e23

data/README.md

@@ -1,24 +1,24 @@
 # fluent-plugin-quota-throttle
 
-[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/rubrikinc/fluent-plugin-throttle/blob/master/LICENSE) ![Rake Test](https://github.com/rubrikinc/fluent-plugin-quota-throttle/actions/workflows/ruby.yml/badge.svg)
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/rubrikinc/fluent-plugin-throttle/blob/master/LICENSE) ![Rake Test](https://github.com/rubrikinc/fluent-plugin-quota-throttle/actions/workflows/ruby.yml/badge.svg) [![Gem Version](https://badge.fury.io/rb/fluent-plugin-quota-throttle.svg)](https://badge.fury.io/rb/fluent-plugin-quota-throttle)
 
 A sentry plugin to throttle logs based on well defined quotas. Logs are grouped by configurable keys. When
 a group exceeds a configuration rate, logs are dropped for this group.
 
 ## Installation
 
-TBD once the plugin is published
+install with `gem` or td-agent provided command as:
 
-[//]: # (install with `gem` or td-agent provided command as:)
 
-[//]: # ()
-[//]: # (```bash)
+```bash
 
-[//]: # (# for fluentd)
+# for fluentd
 
-[//]: # ($ gem install fluent-plugin-quota-throttle)
+$ gem install fluent-plugin-quota-throttle
+[OR]
+$ fluent-gem install fluent-plugin-quota-throttle
 
-[//]: # (```)
+```
 
 ## Configuration
 
@@ -78,7 +78,7 @@ This is the period of of time over which `bucket_size` applies. This should be g
 
 Either `drop` or `reemit`.
 
-When a group exceeds its rate limit, logs are either dropped or re-emitted with a new tag `<tag>.secondary`
+When a group exceeds its rate limit, logs are either dropped or re-emitted with a new tag `secondary.<tag>`
 
 
 
@@ -96,7 +96,7 @@ the delay between every repetition.
 <filter **>
   @type quota_throttle
   @path /etc/fluentd/quota_throttle.yaml
-  @warning_delay 30
+  @warning_delay 30s
 </filter>
 ```
 ```yaml
@@ -108,7 +108,7 @@ quotas:
   match_by:
     group1.a: value1
   bucket_size: 100
-  duration: 60
+  duration: 60s
   action: drop
 - name: quota2
   description: second quota
@@ -119,7 +119,7 @@ quotas:
     group1.a: value2
     group1.b: value3
   bucket_size: 200
-  duration: 120
+  duration: 2m
   action: reemit
 - name: quota3
   description: third quota
@@ -130,14 +130,14 @@ quotas:
       group2: value2
       group3: value3
   bucket_size: 300
-  duration: 180
+  duration: 180s
   action: drop
 default:
   description: default quota
   group_by:
     - group1.a
   bucket_size: 300
-  duration: 180
+  duration: 3m
   action: reemit
 ```
 

data/fluent-plugin-quota-throttle.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-quota-throttle"
-  spec.version       = "0.0.2"
+  spec.version       = "0.0.3"
   spec.authors       = ["Athish Pranav D", "Dipendra Singh", "Rubrik Inc."]
   spec.email         = ["athish.pranav@rubrik.com", "Dipendra.Singh@rubrik.com"]
   spec.summary       = %q{Fluentd filter for throttling logs based on a configurable quotas.}

data/lib/fluent/plugin/config_parser.rb

@@ -16,7 +16,7 @@ module ConfigParser
   #   +action+: (String) The action to take when the quota is reached. Must be one of the predefined actions in @@allowed_actions.
   class Quota
 
-    attr_accessor :name, :desc, :group_by, :match_by, :bucket_size, :duration, :action
+    attr_reader :name, :desc, :group_by, :match_by, :bucket_size, :duration, :action
 
     @@allowed_actions = Set["drop", "reemit"]
 

data/lib/fluent/plugin/filter_quota_throttle.rb

@@ -104,7 +104,7 @@ module Fluent::Plugin
     #   +timestamp+: (Time) The timestamp of the record
     def quota_breached(tag, timestamp, record, bucket, quota)
       if bucket.last_warning.nil? || Time.now - bucket.last_warning > @warning_delay
-        log.warn "Quota breached for group #{bucket.group} in quota #{quota.name}"
+        log.warn "Quota breached for {group: #{bucket.group}, quota: #{quota.name}, total_logs: #{bucket.bucket_count_total}, limit: #{bucket.bucket_limit}, current_rate: #{bucket.approx_rate_per_second}}"
         bucket.last_warning = Time.now
       end
       case quota.action

data/lib/fluent/plugin/rate_limiter.rb

@@ -5,7 +5,9 @@ module RateLimiter
   ##
   # Bucket class, contains the rate limiting logic for each group
   # Attributes:
+  #   +group+: Group for which the bucket is created
   #   +bucket_count+: Number of requests in the bucket
+  #   +bucket_count_total+: Number of requests in the bucket including the dropped requests
   #   +bucket_last_reset+: Time when the bucket was last reset
   #   +approx_rate_per_second+: Approximate rate of requests per second
   #   +rate_last_reset+: Time when the rate was last reset
@@ -16,12 +18,13 @@ module RateLimiter
   #   +bucket_period+: Time period for the bucket
   #   +rate_limit+: Maximum number of requests allowed per second
   class Bucket
-    attr_accessor :bucket_count, :bucket_last_reset, :approx_rate_per_second, :rate_last_reset, :curr_count, :last_warning
+    attr_accessor :bucket_count, :bucket_count_total, :bucket_last_reset, :approx_rate_per_second, :rate_last_reset, :curr_count, :last_warning
     attr_reader :bucket_limit, :bucket_period, :rate_limit, :timeout_s, :group
     def initialize( group, bucket_limit, bucket_period)
       now = Time.now
       @group = group
       @bucket_count = 0
+      @bucket_count_total = 0
       @bucket_last_reset = now
       @approx_rate_per_second = 0
       @rate_last_reset = now
@@ -43,6 +46,7 @@ module RateLimiter
       end
       now = Time.now
       @curr_count += 1
+      @bucket_count_total += 1
       time_lapsed = now - @rate_last_reset
 
       if time_lapsed.to_i >= 1
@@ -80,6 +84,7 @@ module RateLimiter
       now = Time.now
       unless @bucket_count == -1 && @approx_rate_per_second > @rate_limit
         @bucket_count = 0
+        @bucket_count_total = 0
         @bucket_last_reset = now
       end
     end
@@ -99,7 +104,7 @@ module RateLimiter
     #   +group+: Group for which the bucket is required
     #   +quota+: Quota object containing the bucket size and duration
     def get_bucket(group, quota)
-      @buckets[group] = @buckets.delete(group) || Bucket.new( group, quota.bucket_size, quota.duration)
+      @buckets[[group, quota.name]] = @buckets.delete([group, quota.name]) || Bucket.new( group, quota.bucket_size, quota.duration)
     end
 
     # Cleans the buckets that have expired

data/test/modules/rate_limiter_test.rb

@@ -27,6 +27,8 @@ class TestBucket < Minitest::Test
   def test_bucket_allow_full
     11.times { @bucket.allow }
     assert_equal false, @bucket.allow
+    assert_equal -1, @bucket.bucket_count
+    assert_equal 12, @bucket.bucket_count_total
   end
 
   def test_reset_bucket
@@ -63,10 +65,10 @@ class TestBucketStore < Minitest::Test
     group2 = "value2"
     @bucket_store.get_bucket(group2, @quota)
     lru_group, lru_counter = @bucket_store.instance_variable_get(:@buckets).first
-    assert_equal group1, lru_group
+    assert_equal [group1, @quota.name], lru_group
     sleep(5)
     @bucket_store.clean_buckets
     lru_group, lru_counter = @bucket_store.instance_variable_get(:@buckets).first
-    assert_equal group2, lru_group
+    assert_equal [group2, @quota.name], lru_group
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-quota-throttle
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Athish Pranav D
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-14 00:00:00.000000000 Z
+date: 2024-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler