uw-fluent-plugin-throttle 0.0.4.pre.dev

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8997cb3815d33b40957a5d90e394112b42d078eb0c87775f9909756a63cd22c8
+  data.tar.gz: 2a8e0ddb0ef82a8b94b90537fe3ccaa5f8ac77766d4c96ae264dba2ab64472e8
+SHA512:
+  metadata.gz: f3dd854ecf6cadbcc2c3f373d5732f32dfa74832c8ac9c1005df5b2b15011f3463038ef5308fae28dc6df44114dc049e6c18db855bba031b9e55ed79f78605f5
+  data.tar.gz: e186e9bfcbb4f8e89a98b45ba05997191063ac3f8d7e42730ba6c9ae09568a7c77b6e7f0b190eaf2fbda44a947d162c491454119ee333c04970a844cdb8bf7ed

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+os: linux
+sudo: false
+script: bundle exec rake test

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'pry'
+gem 'pry-byebug'

data/Gemfile.lock

@@ -0,0 +1,89 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-throttle (0.0.3)
+      fluentd (~> 1.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    appraisal (2.2.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (10.0.2)
+    coderay (1.1.2)
+    cool.io (1.5.3)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    dig_rb (1.0.1)
+    fluentd (1.2.3)
+      cool.io (>= 1.4.5, < 2.0.0)
+      dig_rb (~> 1.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 0.7.0, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    hashdiff (0.3.7)
+    http_parser.rb (0.6.0)
+    maxitest (3.1.0)
+      minitest (>= 5.0.0, < 5.12.0)
+    metaclass (0.0.4)
+    method_source (0.9.0)
+    minitest (5.11.3)
+    mocha (1.6.0)
+      metaclass (~> 0.0.1)
+    msgpack (1.2.4)
+    power_assert (1.1.3)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.6.0)
+      byebug (~> 10.0)
+      pry (~> 0.10)
+    public_suffix (3.0.2)
+    rake (12.3.1)
+    safe_yaml (1.0.4)
+    serverengine (2.0.7)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    single_cov (1.1.0)
+    strptime (0.2.3)
+    test-unit (3.2.8)
+      power_assert
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2018.5)
+      tzinfo (>= 1.0.0)
+    webmock (3.4.2)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  appraisal (~> 2.2)
+  bundler (~> 1.16)
+  fluent-plugin-throttle!
+  maxitest
+  mocha
+  pry
+  pry-byebug
+  rake (~> 12.3)
+  single_cov
+  test-unit (~> 3.2)
+  webmock (~> 3.3)
+
+BUNDLED WITH
+   1.16.2

data/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,138 @@
+# fluent-plugin-throttle
+
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/rubrikinc/fluent-plugin-throttle/blob/master/LICENSE)
+
+A sentry pluging to throttle logs. Logs are grouped by a configurable key. When
+a group exceeds a configuration rate, logs are dropped for this group.
+
+## Installation
+
+install with `gem` or td-agent provided command as:
+
+```bash
+# for fluentd
+$ gem install fluent-plugin-throttle
+```
+
+## Usage
+
+```xml
+<filter **>
+  @type throttle
+  group_key kubernetes.container_name
+  group_bucket_period_s   60
+  group_bucket_limit    6000
+  group_reset_rate_s     100
+</filter>
+```
+
+## Configuration
+
+#### group\_key
+
+Default: `kubernetes.container_name`.
+
+Used to group logs. Groups are rate limited independently.
+
+A dot indicates a key within a sub-object. As an example, in the following log,
+the group key resolve to "random":
+```
+{"level": "error", "msg": "plugin test", "kubernetes": { "container_name": "random" } }
+```
+
+Multiple groups can be specified using the fluentd config array syntax, 
+e.g. `kubernetes.container_name,kubernetes.pod_name`, in which case each unique pair
+of key values are rate limited independently.
+
+If the group cannot be resolved, an anonymous (`nil`) group is used for rate limiting.
+
+#### group\_bucket\_period\_s
+
+Default: `60` (60 second).
+
+This is the period of of time over which `group_bucket_limit` applies.
+
+#### group\_bucket\_limit
+
+Default: `6000` (logs per `group_bucket_period_s`).
+
+Maximum number logs allowed per groups over the period of `group_bucket_period_s`.
+
+This translate to a log rate of `group_bucket_limit/group_bucket_period_s`.
+When a group exceeds this rate, logs from this group are dropped.
+
+For example, the default is 6000/60s, making for a rate of 100 logs per
+seconds.
+
+Note that this is not expressed as a rate directly because there is a
+difference between the overall rate and the distribution of logs over a period
+time. For example, a burst of logs in the middle of a minute bucket might not
+exceed the average rate of the full minute.
+
+Consider `60/60s`, 60 logs over a minute, versus `1/1s`, 1 log per second.
+Over a minute, both will emit a maximum of 60 logs. Limiting to a rate of 60
+logs per minute. However `60/60s` will readily emit 60 logs within the first
+second then nothing for the remaining 59 seconds. While the `1/1s` will only
+emit the first log of every second.
+
+#### group\_drop\_logs
+
+Default: `true`.
+
+When a group reaches its limit, logs will be dropped from further processing
+if this value is true (set by default). To prevent the logs from being dropped
+and only receive a warning message when rate limiting would have occurred, set
+this value for false. This can be useful to fine-tune your group bucket limits
+before dropping any logs.
+
+#### group\_reset\_rate\_s
+
+Default: `group_bucket_limit/group_bucket_period_s` (logs per `group_bucket_period_s`).
+Maximum: `group_bucket_limit/group_bucket_period_s`.
+
+After a group has exceeded its bucket limit, logs are dropped until the rate
+per second falls below or equal to `group_reset_rate_s`.
+
+The default value is `group_bucket_limits/group_bucket_period_s`. For example
+for 3600 logs per hour, the reset will defaults to `3600/3600s = 1/s`, one log
+per second.
+
+Taking the example `3600 log/hour` with the default reset rate of `1 log/s`
+further:
+
+ - Let's say we have a period of 10 hours.
+ - During the first hour, 2 logs/s are produced. After 30 minutes, the hourly
+   bucket has reached its limit, and logs are dropped. At this point the rate
+   is still 2 logs/s for the remaining 30 minutes.
+ - Because the last hour finished on 2 logs/s, which is higher that the
+   `1 log/s` reset, all logs are still dropped when starting the second hour. The
+   bucket limit is left untouched since nothing is being emitted.
+ - Now, at 2 hours and 30 minutes, the log rate halves to `1 log/s`, which is
+   equal to the reset rate. Logs are emitted again, counting toward the bucket
+   limit as normal. Allowing up to 3600 logs for the last 30 minutes of the second
+   hour.
+
+Because this could allow for some instability if the log rate hovers around the
+`group_bucket_limit/group_bucket_period_s` rate, it is possible to set a
+different reset rate.
+
+Note that a value of `0` effectively means the plugin will drops logs forever
+after a single breach of the limit until the next restart of fluentd.
+
+A value of `-1` disables the feature.
+
+#### group\_warning\_delay\_s
+
+Default: `10` (seconds).
+
+When a group reaches its limit and as long as it is not reset, a warning
+message with the current log rate of the group is emitted repeatedly. This is
+the delay between every repetition.
+
+## License
+
+Apache License, Version 2.0
+
+## Copyright
+
+Copyright © 2018 ([Rubrik Inc.](https://www.rubrik.com))

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+task default: [:test]
+Rake::TestTask.new do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/**/*_test.rb']
+  test.options = '-v'
+end

data/fluent-plugin-throttle.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "uw-fluent-plugin-throttle"
+  spec.version       = "0.0.4-dev"
+  spec.authors       = ["François-Xavier Bourlet", "UtilityWarehouse"]
+  spec.email         = ["fx.bourlet@rubrik.com", "infra@utilitywarehouse.co.uk"]
+  spec.summary       = %q{Fluentd filter for throttling logs based on a configurable key.}
+  spec.homepage      = "https://github.com/utilitywarehouse/fluent-plugin-throttle"
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 12.3"
+  spec.add_development_dependency "webmock", "~> 3.3"
+  spec.add_development_dependency "test-unit", "~> 3.2"
+  spec.add_development_dependency "appraisal", "~> 2.2"
+  spec.add_development_dependency "mocha"
+  spec.add_development_dependency "maxitest"
+  spec.add_development_dependency "single_cov"
+
+  spec.add_runtime_dependency "fluentd", "~> 1.1"
+end

data/lib/fluent/plugin/filter_throttle.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+require 'fluent/plugin/filter'
+
+module Fluent::Plugin
+  class ThrottleFilter < Filter
+    Fluent::Plugin.register_filter('throttle', self)
+
+    desc "Used to group logs. Groups are rate limited independently"
+    config_param :group_key, :array, :default => ['kubernetes.container_name']
+
+    desc <<~DESC
+      This is the period of of time over which group_bucket_limit applies
+    DESC
+    config_param :group_bucket_period_s, :integer, :default => 60
+
+    desc <<~DESC
+      Maximum number logs allowed per groups over the period of
+      group_bucket_period_s
+    DESC
+    config_param :group_bucket_limit, :integer, :default => 6000
+
+    desc "Whether to drop logs that exceed the bucket limit or not"
+    config_param :group_drop_logs, :bool, :default => true
+
+    desc <<~DESC
+      After a group has exceeded its bucket limit, logs are dropped until the
+      rate per second falls below or equal to group_reset_rate_s.
+    DESC
+    config_param :group_reset_rate_s, :integer, :default => nil
+
+    desc <<~DESC
+      When a group reaches its limit and as long as it is not reset, a warning
+      message with the current log rate of the group is emitted repeatedly.
+      This is the delay between every repetition.
+    DESC
+    config_param :group_warning_delay_s, :integer, :default => 10
+
+    Group = Struct.new(
+      :rate_count,
+      :rate_last_reset,
+      :aprox_rate,
+      :bucket_count,
+      :bucket_last_reset,
+      :last_warning)
+
+    def configure(conf)
+      super
+
+      @group_key_paths = group_key.map { |key| key.split(".") }
+
+      raise "group_bucket_period_s must be > 0" \
+        unless @group_bucket_period_s > 0
+
+      raise "group_bucket_limit must be > 0" \
+        unless @group_bucket_limit > 0
+
+      @group_rate_limit = (@group_bucket_limit / @group_bucket_period_s)
+
+      @group_reset_rate_s = @group_rate_limit \
+        if @group_reset_rate_s == nil
+
+      raise "group_reset_rate_s must be >= -1" \
+        unless @group_reset_rate_s >= -1
+      raise "group_reset_rate_s must be <= group_bucket_limit / group_bucket_period_s" \
+        unless @group_reset_rate_s <= @group_rate_limit
+
+      raise "group_warning_delay_s must be >= 1" \
+        unless @group_warning_delay_s >= 1
+    end
+
+    def start
+      super
+
+      @counters = {}
+    end
+
+    def shutdown
+      log.debug("counters summary: #{@counters}")
+      super
+    end
+
+    def filter(tag, time, record)
+      now = Time.now
+      rate_limit_exceeded = @group_drop_logs ? nil : record # return nil on rate_limit_exceeded to drop the record
+      group = extract_group(record)
+      counter = (@counters[group] ||= Group.new(0, now, 0, 0, now, nil))
+      counter.rate_count += 1
+
+      since_last_rate_reset = now - counter.rate_last_reset
+      if since_last_rate_reset >= 1
+        # compute and store rate/s at most every seconds.
+        counter.aprox_rate = (counter.rate_count / since_last_rate_reset).round()
+        counter.rate_count = 0
+        counter.rate_last_reset = now
+      end
+
+      if (now.to_i / @group_bucket_period_s) \
+          > (counter.bucket_last_reset.to_i / @group_bucket_period_s)
+        # next time period reached.
+
+        # wait until rate drops back down (if enabled).
+        if counter.bucket_count == -1 and @group_reset_rate_s != -1
+          if counter.aprox_rate < @group_reset_rate_s
+            log_rate_back_down(now, group, counter)
+          else
+            log_rate_limit_exceeded(now, group, counter)
+            return rate_limit_exceeded
+          end
+        end
+
+        # reset counter for the rest of time period.
+        counter.bucket_count = 0
+        counter.bucket_last_reset = now
+      else
+        # if current time period credit is exhausted, drop the record.
+        if counter.bucket_count == -1
+          log_rate_limit_exceeded(now, group, counter)
+          return rate_limit_exceeded
+        end
+      end
+
+      counter.bucket_count += 1
+
+      # if we are out of credit, we drop logs for the rest of the time period.
+      if counter.bucket_count > @group_bucket_limit
+        log_rate_limit_exceeded(now, group, counter)
+        counter.bucket_count = -1
+        return rate_limit_exceeded
+      end
+
+      record
+    end
+
+    private
+
+    def extract_group(record)
+      @group_key_paths.map do |key_path|
+        record.dig(*key_path) || record.dig(*key_path.map(&:to_sym))
+      end
+    end
+
+    def log_rate_limit_exceeded(now, group, counter)
+      emit = counter.last_warning == nil ? true \
+        : (now - counter.last_warning) >= @group_warning_delay_s
+      if emit
+        log.warn("rate exceeded", log_items(now, group, counter))
+        counter.last_warning = now
+      end
+    end
+
+    def log_rate_back_down(now, group, counter)
+      log.info("rate back down", log_items(now, group, counter))
+    end
+
+    def log_items(now, group, counter)
+      since_last_reset = now - counter.bucket_last_reset
+      rate = since_last_reset > 0 ? (counter.bucket_count / since_last_reset).round : Float::INFINITY
+      aprox_rate = counter.aprox_rate
+      rate = aprox_rate if aprox_rate > rate
+
+      {'group_key': group,
+       'rate_s': rate,
+       'period_s': @group_bucket_period_s,
+       'limit': @group_bucket_limit,
+       'rate_limit_s': @group_rate_limit,
+       'reset_rate_s': @group_reset_rate_s}
+    end
+  end
+end

data/test/fluent/plugin/filter_throttle_test.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+require_relative '../../helper'
+require 'fluent/plugin/filter_throttle'
+
+SingleCov.covered!
+
+describe Fluent::Plugin::ThrottleFilter do
+  include Fluent::Test::Helpers
+
+  before do
+    Fluent::Test.setup
+  end
+
+  after do
+    if instance_variable_defined?(:@driver)
+      assert @driver.error_events.empty?, "Errors detected: " + @driver.error_events.map(&:inspect).join("\n")
+    end
+  end
+
+  def create_driver(conf='')
+    @driver = Fluent::Test::Driver::Filter.new(Fluent::Plugin::ThrottleFilter).configure(conf)
+  end
+
+  describe '#configure' do
+    it 'raises on invalid group_bucket_limit' do
+      assert_raises { create_driver("group_bucket_limit 0") }
+    end
+
+    it 'raises on invalid group_bucket_period_s' do
+      assert_raises { create_driver("group_bucket_period_s 0") }
+    end
+
+    it 'raises on invalid group_reset_rate_s' do
+      assert_raises { create_driver("group_bucket_limit 10\ngroup_bucket_period_s 10\ngroup_reset_rate_s 2") }
+    end
+
+    it 'raises on invalid group_reset_rate_s' do
+      assert_raises { create_driver("group_bucket_limit 10\ngroup_bucket_period_s 10\ngroup_reset_rate_s -2") }
+    end
+
+    it 'raises on invalid group_warning_delay_s' do
+      assert_raises { create_driver("group_warning_delay_s 0") }
+    end
+  end
+
+  describe '#filter' do
+    it 'throttles per group key' do
+      driver = create_driver <<~CONF
+        group_key "group"
+        group_bucket_period_s 1
+        group_bucket_limit 5
+      CONF
+
+      driver.run(default_tag: "test") do
+        driver.feed([[event_time, {"msg": "test", "group": "a"}]] * 10)
+        driver.feed([[event_time, {"msg": "test", "group": "b"}]] * 10)
+      end
+
+      groups = driver.filtered_records.group_by { |r| r[:group] }
+      assert_equal(5, groups["a"].size)
+      assert_equal(5, groups["b"].size)
+    end
+
+    it 'allows composite group keys' do
+      driver = create_driver <<~CONF
+        group_key "group1,group2"
+        group_bucket_period_s 1
+        group_bucket_limit 5
+      CONF
+
+      driver.run(default_tag: "test") do
+        driver.feed([[event_time, {"msg": "test", "group1": "a", "group2": "b"}]] * 10)
+        driver.feed([[event_time, {"msg": "test", "group1": "b", "group2": "b"}]] * 10)
+        driver.feed([[event_time, {"msg": "test", "group1": "c"}]] * 10)
+        driver.feed([[event_time, {"msg": "test", "group2": "c"}]] * 10)
+      end
+
+      groups = driver.filtered_records.group_by { |r| r[:group1] }
+      groups.each { |k, g| groups[k] = g.group_by { |r| r[:group2] } }
+
+      assert_equal(5, groups["a"]["b"].size)
+      assert_equal(5, groups["b"]["b"].size)
+      assert_equal(5, groups["c"][nil].size)
+      assert_equal(5, groups[nil]["c"].size)
+    end
+
+    it 'drops until rate drops below group_reset_rate_s' do
+      driver = create_driver <<~CONF
+        group_bucket_period_s 60
+        group_bucket_limit 180
+        group_reset_rate_s 2
+      CONF
+
+      logs_per_sec = [4, 4, 2, 1, 2]
+
+      driver.run(default_tag: "test") do
+        (0...logs_per_sec.size*60).each do |i|
+          Time.stubs(now: Time.at(i))
+          min = i / 60
+
+          driver.feed([[event_time, min: min]] * logs_per_sec[min])
+        end
+      end
+
+      groups = driver.filtered_records.group_by { |r| r[:min] }
+      messages_per_minute = []
+      (0...logs_per_sec.size).each do |min|
+        messages_per_minute[min] = groups.fetch(min, []).size
+      end
+
+      assert_equal [
+        180, # hits the limit in the first minute
+        0,   # still >= group_reset_rate_s
+        0,   # still >= group_reset_rate_s
+        59,  # now < group_reset_rate_s, stop dropping logs (except the first log is dropped)
+        120  # > group_reset_rate_s is okay now because haven't hit the limit
+      ], messages_per_minute
+    end
+
+
+    it 'does not throttle when in log only mode' do
+      driver = create_driver <<~CONF
+        group_bucket_period_s 2
+        group_bucket_limit 4
+        group_reset_rate_s 2
+        group_drop_logs false
+      CONF
+
+      records_expected = 0
+      driver.run(default_tag: "test") do
+        (0...10).each do |i|
+          Time.stubs(now: Time.at(i))
+          count = [1,8 - i].max
+          records_expected += count
+          driver.feed((0...count).map { |j| [event_time, msg: "test#{i}-#{j}"] }) # * count)
+        end
+      end
+
+      assert_equal records_expected, driver.filtered_records.size
+      assert driver.logs.any? { |log| log.include?('rate exceeded') }
+      assert driver.logs.any? { |log| log.include?('rate back down') }
+    end
+  end
+
+  describe 'logging' do
+    it 'logs when rate exceeded once per group_warning_delay_s' do
+      driver = create_driver <<~CONF
+        group_bucket_period_s 2
+        group_bucket_limit 2
+        group_warning_delay_s 3
+      CONF
+
+      logs_per_sec = 4
+
+      driver.run(default_tag: "test") do
+        (0...10).each do |i|
+          Time.stubs(now: Time.at(i))
+          driver.feed([[event_time, msg: "test"]] * logs_per_sec)
+        end
+      end
+
+      assert_equal 4, driver.logs.select { |log| log.include?('rate exceeded') }.size
+    end
+
+    it 'logs when rate drops below group_reset_rate_s' do
+      driver = create_driver <<~CONF
+        group_bucket_period_s 2
+        group_bucket_limit 4
+        group_reset_rate_s 2
+      CONF
+
+      driver.run(default_tag: "test") do
+        (0...10).each do |i|
+          Time.stubs(now: Time.at(i))
+          driver.feed([[event_time, msg: "test"]] * [1,8 - i].max)
+        end
+      end
+
+      assert driver.logs.any? { |log| log.include?('rate back down') }
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,12 @@
+require 'bundler/setup'
+require 'single_cov'
+
+SingleCov.setup :minitest
+
+require 'fluent/test'
+require 'fluent/test/driver/input'
+require 'fluent/test/driver/filter'
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
+require 'maxitest/autorun'
+require 'mocha/minitest'

metadata

@@ -0,0 +1,185 @@
+--- !ruby/object:Gem::Specification
+name: uw-fluent-plugin-throttle
+version: !ruby/object:Gem::Version
+  version: 0.0.4.pre.dev
+platform: ruby
+authors:
+- François-Xavier Bourlet
+- UtilityWarehouse
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-11-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: maxitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: single_cov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+description: 
+email:
+- fx.bourlet@rubrik.com
+- infra@utilitywarehouse.co.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-throttle.gemspec
+- lib/fluent/plugin/filter_throttle.rb
+- test/fluent/plugin/filter_throttle_test.rb
+- test/helper.rb
+homepage: https://github.com/utilitywarehouse/fluent-plugin-throttle
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Fluentd filter for throttling logs based on a configurable key.
+test_files:
+- test/fluent/plugin/filter_throttle_test.rb
+- test/helper.rb