fluent-plugin-throttle 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a6e438f87f8be6572c11b82e0dce8c493b3ed92d
+  data.tar.gz: e6cb87f9b709ea05d72e8244c72b2daf814ed741
+SHA512:
+  metadata.gz: 814306a6ae6eee257ea97734a101f5edd0daacf6754f2dbc7e031f3e8be01dbe05bc0d6592dce6911f43d1c9b082f4597fc43acc687c4ff7180d51eb04190d07
+  data.tar.gz: 72ea81c1dd748d0c5ac0650c6add62d4eb2302f1d4e5704b886161cc6fcd00ddc123948e9c2ce58ea69fbcc14d85773361e1a012851e12c4cbc4e273cdb734fb

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/README.md

@@ -0,0 +1,117 @@
+# fluent-plugin-throttle
+
+A sentry pluging to throttle logs. Logs are grouped by a configurable key. When
+a group exceeds a configuration rate, logs are dropped for this group.
+
+## Installation
+
+install with `gem` or td-agent provided command as:
+
+```bash
+# for fluentd
+$ gem install fluent-plugin-throttle
+```
+
+## Usage
+
+```xml
+<filter>
+	...
+</filter>
+```
+
+## Configuration
+
+### group\_key
+
+Default: `kubernetes.container_name`.
+
+Used to group logs. Groups are rate limited independently.
+
+A dot indicates a key within a sub-object. As an example, in the following log,
+the group key resolve to "random":
+```
+{"level": "error", "msg": "plugin test", "kubernetes": { "container_name": "random" } }
+```
+
+If the group cannot be resolved, am anonymous group is used.
+
+## group\_bucket\_period\_s
+
+Default: `60` (60 second).
+
+This is the period of of time over which `group_bucket_limit` applies.
+
+## group\_bucket\_limit
+
+Default: `6000` (logs per `group_bucket_period_s`).
+
+Maximum number logs allowed per groups over the period of `group_bucket_period_s`.
+
+This translate to a log rate of `group_bucket_limit/group_bucket_period_s`.
+When a group exceeds this rate, logs from this group are dropped.
+
+For example, the default is 6000/60s, making for a rate of 100 logs per
+seconds.
+
+Note that this is not expressed as a rate directly because there is a
+difference between the overall rate and the distribution of logs over a period
+time. For example, a burst of logs in the middle of a minute bucket might not
+exceed the average rate of the full minute.
+
+Consider `60/60s`, 60 logs over a minute, versus `1/1s`, 1 log per second.
+Over a minute, both will emit a maximum of 60 logs. Limiting to a rate of 60
+logs per minute. However `60/60s` will readily emit 60 logs within the first
+second then nothing for the remaining 59 seconds. While the `1/1s` will only
+emit the first log of every second.
+
+## group\_reset\_rate\_s
+
+Default: `group_bucket_limit/group_bucket_period_s` (logs per `group_bucket_period_s`).
+Max: `group_bucket_limit`.
+
+After a group has exceeded its bucket limit, logs are dropped until the rate
+per second falls below or equal to `group_reset_rate_s`.
+
+The default value is `group_bucket_limits/group_bucket_period_s`. For example
+for 3600 logs per hour, the reset will defaults to `3600/3600s = 1/s`, one log
+per second.
+
+Taking the example `3600 log/hour` with the default reset rate of `1 log/s`
+further:
+
+ - Let's say we have a period of 10 hours.
+ - During the first hour, 2 logs/s are produced. After 30 minutes, the hourly
+   bucket has reached its limit, and logs are dropped. At this point the rate
+   is still 2 logs/s for the remaining 30 minutes.
+ - Because the last hour finished on 2 logs/s, which is higher that the
+   `1 log/s` reset, all logs are still dropped when starting the second hour. The
+   bucket limit is left untouched since nothing is being emitted.
+ - Now, at 2 hours and 30 minutes, the log rate halves to `1 log/s`, which is
+   equal to the reset rate. Logs are emitted again, counting toward the bucket
+   limit as normal. Allowing up to 3600 logs for the last 30 minutes of the second
+   hour.
+
+Because this could allow for some instability if the log rate hovers around the
+`group_bucket_limit/group_bucket_period_s` rate, it is possible to set a
+different reset rate.
+
+Note that a value of `0` effectively means the plugin will drops logs forever
+after a single breach of the limit until the next restart of fluentd.
+
+A value of `-1` disables the feature.
+
+## group\_warning\_hz
+
+Default: `0.1` (10 seconds).
+
+When a group reaches its limit and as long as it is not reset, a warning
+message with the current log rate for the group is logged at this frequency.
+
+## Copyright
+
+Copyright © 2018 ([Rubrik Inc.](https://www.rubrik.com))
+
+## License
+
+Apache License, Version 2.0

data/fluent-plugin-throttle.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-throttle"
+  spec.version       = "0.0.1"
+  spec.authors       = ["François-Xavier Bourlet"]
+  spec.email         = ["fx.bourlet@rubrik.com"]
+  spec.summary       = %q{Fluentd filter for throttling logs based on a configurable key.}
+  spec.homepage      = "https://github.com/bombela/fluent-plugin-throttle"
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 12.3"
+  spec.add_development_dependency "webmock", "~> 3.3"
+  spec.add_development_dependency "test-unit", "~> 3.2"
+  spec.add_development_dependency "appraisal", "~> 2.2"
+
+  spec.add_runtime_dependency "fluentd", "~> 1.1"
+end

data/lib/fluent/plugin/filter_throttle.rb

@@ -0,0 +1,139 @@
+require 'fluent/filter'
+
+module Fluent
+  class ThrottleFilter < Filter
+    Fluent::Plugin.register_filter('throttle', self)
+
+    config_param :group_key, :string, :default => 'kubernetes.container_name'
+    config_param :group_bucket_period_s, :integer, :default => 60
+    config_param :group_bucket_limit, :integer, :default => 6000
+    config_param :group_reset_rate_s, :integer, :default => nil
+    config_param :warning_hz, :float, :default => 0.1
+
+    Bucket = Struct.new(:emitted, :last_reset)
+    Group = Struct.new(
+      :rate_count,
+      :rate_last_reset,
+      :rate,
+      :bucket_count,
+      :bucket_last_reset,
+      :last_warning)
+
+    def configure(conf)
+      super
+
+      @group_key_path = group_key.split(".")
+
+      raise "group_bucket_period_s must be > 0" \
+        unless @group_bucket_period_s > 0
+
+      raise "group_bucket_limit must be > 0" \
+        unless @group_bucket_limit > 0
+
+      @group_rate_limit = (@group_bucket_limit / @group_bucket_period_s)
+
+      @group_reset_rate_s = @group_rate_limit \
+        if @group_reset_rate_s == nil
+
+      raise "group_reset_rate_s must be >= -1" \
+        unless @group_reset_rate_s >= -1
+      raise "group_reset_rate_s must be <= group_bucket_limit / group_bucket_period_s" \
+        unless @group_reset_rate_s <= @group_rate_limit
+
+      @warning_delay = (1.0 / @warning_hz)
+    end
+
+    def start
+      super
+
+      @counters = Hash.new()
+    end
+
+    def shutdown
+      $log.info("counters summary: #{@counters}")
+      super
+    end
+
+    def filter(tag, time, record)
+      now = Time.now
+      group = extract_group(record)
+      counter = @counters.fetch(group, nil)
+      counter = @counters[group] = Group.new(
+        0, now, 0, 0, now, nil) if counter == nil
+
+      counter.rate_count += 1
+
+      since_last_rate_reset = now - counter.rate_last_reset
+      if since_last_rate_reset >= 1
+        # compute and store rate/s at most every seconds.
+        counter.rate = (counter.rate_count / since_last_rate_reset).round()
+        counter.rate_count = 0
+        counter.rate_last_reset = now
+      end
+
+      if (now.to_i / @group_bucket_period_s) \
+          > (counter.bucket_last_reset.to_i / @group_bucket_period_s)
+        # next time period reached, reset limit.
+
+        if counter.bucket_count == -1 and @group_reset_rate_s != -1
+          # wait until rate drops back down if needed.
+          if counter.rate < @group_reset_rate_s
+            log_rate_back_down(now, group, counter)
+          else
+            since_last_warning = now - counter.last_warning
+            if since_last_warning > @warning_delay
+              log_rate_limit_exceeded(now, group, counter)
+              counter.last_warning = now
+            end
+            return nil
+          end
+        end
+
+        counter.bucket_count = 0
+        counter.bucket_last_reset = now
+      end
+
+      if counter.bucket_count == -1
+        return nil
+      end
+
+      counter.bucket_count += 1
+
+      if counter.bucket_count > @group_bucket_limit
+        log_rate_limit_exceeded(now, group, counter)
+        counter.last_warning = now
+        counter.bucket_count = -1
+        return nil
+      end
+
+      record
+    end
+
+    def extract_group(record)
+      record.dig(*@group_key_path)
+    end
+
+    def log_rate_limit_exceeded(now, group, counter)
+      $log.warn("rate exceeded", log_items(now, group, counter))
+    end
+
+    def log_rate_back_down(now, group, counter)
+      $log.info("rate back down", log_items(now, group, counter))
+    end
+
+    def log_items(now, group, counter)
+      rate = counter.rate
+      if rate == 0
+        since_last_reset = now - counter.bucket_last_reset
+        rate = (counter.bucket_count / since_last_reset).round()
+      end
+
+      {'group_key': group,
+       'rate_s': rate,
+       'period_s': @group_bucket_period_s,
+       'limit': @group_bucket_limit,
+       'rate_limit_s': @group_rate_limit,
+       'reset_rate_s': @group_reset_rate_s}
+    end
+  end
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-throttle
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- François-Xavier Bourlet
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-03-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+description: 
+email:
+- fx.bourlet@rubrik.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- README.md
+- fluent-plugin-throttle.gemspec
+- lib/fluent/plugin/filter_throttle.rb
+homepage: https://github.com/bombela/fluent-plugin-throttle
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.1
+signing_key: 
+specification_version: 4
+summary: Fluentd filter for throttling logs based on a configurable key.
+test_files: []