logstash-filter-throttle-prop 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1de866a8a0676952e1ae664db261ae6a2ae830fe
+  data.tar.gz: 760ed358226f05c5b7ae5e2d96513dabfc12992c
+SHA512:
+  metadata.gz: 961256d37c2b2b6e9deb5c330bdc1a4861ac6cb3d6efb64593de7e835a5fb4f63022a1499ea89ed0e4d0bf8a74ab69f11ca757dbb1eea5ef2f47d1a67f5d2f53
+  data.tar.gz: 6d9e0dc7cb4a113084f1049ee64c2563a455bf02503c53a849e870b5918a41190ad43d0f80d4a86e8d1d252c015a674293ba54876f1e209748f13f5e60839fae

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.1.0
+  - Plugin created with the logstash plugin generator

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* your_username - your_username@example.com
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-filter-throttle-prop
+Example filter plugin. This should help bootstrap your effort to write your own filter plugin!

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+

data/LICENSE

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,69 @@
+# Logstash Filter Throttle Prop
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+The throttle-prop-filter is to throttle events per a dynamic property. The filter is configured with a key, a value and a limit. The key is used to group the values. If the values with a different content are inserted more than limit, it will be filtered.
+
+
+## How to install
+
+```sh
+bin/logstash-plugin install logstash-filter-throttle-prop
+```
+
+## Hou to use
+
+For example, if you wanted to throttle events per a property "type",
+and you didn't want to have more than two differents types,
+you would use the configuration:
+
+```ruby
+filter {
+  throttle_prop {
+    key => "fixed_key"
+    value => "%{type}"
+    limit => 2
+    add_tag => [ "throttled" ]
+  }
+}
+```
+
+##### Which would result in:
+   - event 1  { type => 'a' } - not throttled
+   - event 2  { type => 'a' } - not throttled
+   - event 3  { type => 'b' } - not throttled
+   - event 4  { type => 'b' } - not throttled
+   - event 5  { type => 'c' } - throttled (successful filter)
+   - event 6  { type => 'c' } - throttled (successful filter)
+   - event 7  { type => 'a' } - not throttled
+   - event 8  { type => 'b' } - not throttled
+   - event 9  { type => 'c' } - throttled (successful filter)
+   - event 10 { type => 'd' } - throttled (successful filter)
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```

data/lib/logstash/filters/throttle_prop.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The throttle-prop-filter is to throttle events per a dynamic property.
+# The filter is configured with a key, a value and a limit.
+# The key is used to group the values.
+# If the values with a different content are be inserted more
+# than limit, it will be filtered
+#
+#
+# For example, if you wanted to throttle events per a property "type",
+# and you didn't want to have more than two differents types,
+# you would use the configuration:
+#
+# [source, ruby]
+#     key => "fixed_key"
+#     value => "%{type}"
+#     limit => 2
+#
+# Which would result in:
+# ==========================
+#     event 1  { type => 'a' } - not throttled
+#     event 2  { type => 'a' } - not throttled
+#     event 3  { type => 'b' } - not throttled
+#     event 4  { type => 'b' } - not throttled
+#     event 5  { type => 'c' } - throttled (successful filter)
+#     event 6  { type => 'c' } - throttled (successful filter)
+#     event 7  { type => 'a' } - not throttled
+#     event 8  { type => 'b' } - not throttled
+#     event 9  { type => 'c' } - throttled (successful filter)
+#     event 10 { type => 'd' } - throttled (successful filter)
+
+class LogStash::Filters::ThrottleProp < LogStash::Filters::Base
+
+  config_name "throttle_prop"
+
+  config :key, :validate => :string, :required => true
+
+  config :value, :validate => :string, :required => true
+
+  config :limit, :validate => :number, :required => true
+
+  public
+  def register
+    @keys = Hash.new
+  end # def register
+
+  public
+  def filter(event)
+    key = event.sprintf(@key)
+    value = event.sprintf(@value)
+    limit = event.sprintf(@limit).to_i
+
+    if @keys.has_key? @key
+      unless @keys[@key].include? value
+        if @keys[@key].length + 1 > limit
+          filter_matched(event)
+        else
+          @keys[@key].push(value)
+        end
+      end
+    else
+      @keys[@key] = [value]
+    end
+
+  end # def filter
+end # class LogStash::Filters::ThrottleProp

data/logstash-filter-throttle-prop.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-filter-throttle-prop'
+  s.version       = '0.1.0'
+  s.licenses      = ['Apache License (2.0)']
+  s.description   = 'A logstash filter plugin to throttle events per properties'
+  s.summary       = s.description
+  s.homepage      = 'https://github.com/JeanPinzon/logstash-filter-throttle-prop'
+  s.authors       = ['jeanpinzon']
+  s.email         = 'jean.pinzon1@gmail.com'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/filters/throttle_prop_spec.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+require_relative '../spec_helper'
+require "logstash/filters/throttle_prop"
+
+describe LogStash::Filters::ThrottleProp do
+
+  let(:config) do <<-CONFIG
+    filter {
+      throttle_prop {
+        key => "test"
+        value => "%{property}"
+        limit => 2
+        add_tag => [ "throttled" ]
+      }
+    }
+  CONFIG
+  end
+
+  describe "Throttling per property" do
+    events = [{
+      "property" => "a"
+    }, {
+      "property" => "b"
+    }, {
+      "property" => "c"
+    }]
+
+    sample events do
+      expect(subject[0].get("property")).to eq('a')
+      expect(subject[1].get("property")).to eq('b')
+      expect(subject[2].get("property")).to eq('c')
+      expect(subject[2].get("tags")).to eq([ "throttled" ])
+    end
+  end
+
+  describe "Don't throttling equal properties" do
+    events = [{
+      "property" => "a"
+    }, {
+      "property" => "a"
+    }, {
+      "property" => "a"
+    }]
+
+    sample events do
+      expect(subject[0].get("property")).to eq('a')
+      expect(subject[1].get("property")).to eq('a')
+      expect(subject[2].get("property")).to eq('a')
+      expect(subject[2].get("tags")).to eq(nil)
+    end
+  end
+
+  describe "Don't throttling equal properties and throttling differents" do
+    events = [{
+      "property" => "a"
+    }, {
+      "property" => "a"
+    }, {
+      "property" => "b"
+    }, {
+      "property" => "b"
+    }, {
+      "property" => "c"
+    }, {
+      "property" => "c"
+    }, {
+      "property" => "a"
+    }, {
+      "property" => "b"
+    }, {
+      "property" => "c"
+    }, {
+      "property" => "d"
+    }]
+
+    sample events do
+      expect(subject[0].get("property")).to eq('a')
+      expect(subject[1].get("property")).to eq('a')
+      expect(subject[2].get("property")).to eq('b')
+      expect(subject[3].get("property")).to eq('b')
+      expect(subject[4].get("property")).to eq('c')
+      expect(subject[5].get("property")).to eq('c')
+      expect(subject[4].get("tags")).to eq([ "throttled" ])
+      expect(subject[5].get("tags")).to eq([ "throttled" ])
+      expect(subject[6].get("property")).to eq('a')
+      expect(subject[7].get("property")).to eq('b')
+      expect(subject[8].get("property")).to eq('c')
+      expect(subject[8].get("tags")).to eq([ "throttled" ])
+      expect(subject[9].get("property")).to eq('d')
+      expect(subject[9].get("tags")).to eq([ "throttled" ])
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-throttle-prop
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- jeanpinzon
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-05-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A logstash filter plugin to throttle events per properties
+email: jean.pinzon1@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/throttle_prop.rb
+- logstash-filter-throttle-prop.gemspec
+- spec/filters/throttle_prop_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/JeanPinzon/logstash-filter-throttle-prop
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: A logstash filter plugin to throttle events per properties
+test_files:
+- spec/filters/throttle_prop_spec.rb
+- spec/spec_helper.rb