fluent-plugin-conditional_filter 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6cc5c490e9566041f5cd52dcdbccaa2e37db1477
-  data.tar.gz: bd5605aeb0b2ed35045020e9b1a45ab0e9206146
+  metadata.gz: 4e96a9f324a32ecb3745e54e5d249407cea733ce
+  data.tar.gz: 853c3cbe3440a42d81b7a72f3642f2592ac16646
 SHA512:
-  metadata.gz: 9cda736e9cdbddc4e0d4cdec38f9c5e913bec91817c3d4085209d07d0be6164abb11615679e29bdd4b2e326b965ea5ad5bc30ce01c9aba078dde35db84926dca
-  data.tar.gz: d5d9fef02572bc606b5877382b045f1ede9975682a425d314b2d91423da8ab767308512c03b019eda1aac89e7afe677e31ea244ea6e913f63244f8b486df9694
+  metadata.gz: eec6a1bae7039cbdd952e0cd772b584f59e22b4cf9182eff13bcd551383c3d0a880a67954a66f25aeacb811a395c2cb1bd14aed07fe5badfffd299b7d107a7b0
+  data.tar.gz: 0a3dc292804cde6c81b3b19572170ba39eaeca7ff42b83ab0ac2b61eb8eeb75839204fc34e15dbfd9e8af1ce93669abeb709b4d1b66e9c8c572a75263be12cb7

data/README.md

@@ -2,6 +2,57 @@
 
 ## Component
 
+### ConditionalFilter
+
+fluent-plugin-conditional_filter provides a simple filter that filters out key/value pairs that don't satisfy a given condition. This is the filter version of [ConditionalFilterOutput](#conditionalfilteroutput).
+
+## Usage
+
+### Synopsis
+
+If there's such a configuration as below:
+
+```
+<filter test.**>
+  type           conditional
+  key_pattern    @example\.com$
+  condition      10
+  filter         numeric_upward
+</filter>
+```
+
+When the log below reaches:
+
+```
+'test' => {
+  'foo@example.com' => 5,
+  'bar@example.com' => 15,
+  'baz@baz.com'     => 12,
+}
+```
+
+key/value pairs that don't match either `key_pattern` or the condition designated by `condition` and `filter` are filtered out.
+
+```
+'filtered.test' => {
+  'bar@example.com' => 15,
+}
+```
+
+### Params
+
+#### `key_pattern` (required)
+
+Key pattern to check.
+
+#### `condition` (required)
+
+Condition for the filter below.
+
+#### `filter` (required)
+
+Set filtering strategy.
+
 ### ConditionalFilterOutput
 
 fluent-plugin-conditional_filter provides a simple filter that filters out key/value pairs that don't satisfy a given condition.

data/fluent-plugin-conditional_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-conditional_filter'
-  spec.version       = '0.0.3'
+  spec.version       = '0.0.4'
   spec.authors       = ['Kentaro Kuribayashi']
   spec.email         = ['kentarok@gmail.com']
   spec.description   = %q{A fluent plugin that provides conditional filters}
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'fluentd'
+  spec.add_runtime_dependency 'fluentd', [">= 0.14.0", "< 2"]
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'

data/lib/fluent/plugin/conditional_filter_rule.rb

@@ -0,0 +1,30 @@
+require 'fluent/output'
+
+module Fluent
+  module ConditionalFilterRule
+    def filter_record(tag, time, record, plugin)
+      super(tag, time, record) if plugin.is_a?(Fluent::Output)
+      case @filter
+      when 'numeric_upward'
+        filter_record = record.select do |key, value|
+          key.match(@key_pattern_regexp) &&
+            record[key].to_f >= @condition.to_f
+        end
+      when 'numeric_downward'
+        filter_record = record.select do |key, value|
+          key.match(@key_pattern_regexp) &&
+            record[key].to_f <= @condition.to_f
+        end
+      when 'string_match'
+        filter_record = record.select do |key, value|
+          key.match(@key_pattern_regexp) &&
+            record[key].match(Regexp.new(@condition))
+        end
+      else
+        raise ArgumentError.new("[conditional_filter_rule] no such filter: #{filter}")
+      end
+
+      filter_record
+    end
+  end
+end

data/lib/fluent/plugin/filter_conditional.rb

@@ -0,0 +1,24 @@
+require 'fluent/plugin/conditional_filter_rule'
+require 'fluent/plugin/filter'
+
+class Fluent::Plugin::ConditionalFilter < Fluent::Plugin::Filter
+  Fluent::Plugin.register_filter('conditional', self)
+
+  include Fluent::ConditionalFilterRule
+
+  config_param :key_pattern, :string
+  config_param :condition,   :string
+  config_param :filter,      :string
+
+  def configure(conf)
+    super
+
+    @key_pattern_regexp = Regexp.new(key_pattern)
+  end
+
+  def filter(tag, time, record)
+    record = filter_record(tag, time, record, self)
+
+    record if record.any?
+  end
+end

data/lib/fluent/plugin/out_conditional_filter.rb

@@ -1,3 +1,4 @@
+require 'fluent/plugin/conditional_filter_rule'
 class Fluent::ConditionalFilterOutput < Fluent::Output
   Fluent::Plugin.register_output('conditional_filter', self)
 
@@ -7,6 +8,7 @@ class Fluent::ConditionalFilterOutput < Fluent::Output
   end
 
   include Fluent::HandleTagNameMixin
+  include Fluent::ConditionalFilterRule
 
   config_param :key_pattern, :string
   config_param :condition,   :string
@@ -28,7 +30,7 @@ class Fluent::ConditionalFilterOutput < Fluent::Output
   def emit(tag, es, chain)
     es.each do |time, record|
       t = tag.dup
-      record = filter_record(t, time, record)
+      record = filter_record(t, time, record, self)
 
       if record.any?
         router.emit(t, time, record)
@@ -37,33 +39,4 @@ class Fluent::ConditionalFilterOutput < Fluent::Output
 
     chain.next
   end
-
-  private
-
-  def filter_record(tag, time, record)
-    super
-    case filter
-    when 'numeric_upward'
-      filter_record = record.select do |key, value|
-        key.match(@key_pattern_regexp) &&
-        record[key].to_f >= condition.to_f
-      end
-    when 'numeric_downward'
-      filter_record = record.select do |key, value|
-        key.match(@key_pattern_regexp) &&
-        record[key].to_f <= condition.to_f
-      end
-    when 'string_match'
-      filter_record = record.select do |key, value|
-        key.match(@key_pattern_regexp) &&
-        record[key].match(Regexp.new(condition))
-      end
-    else
-      raise ArgumentError.new("[out_conditional_filter] no such filter: #{filter}")
-    end
-
-    filter_record
-  end
 end
-
-

data/spec/lib/filter_conditional_spec.rb

@@ -0,0 +1,220 @@
+require 'spec_helper'
+
+describe Fluent::Plugin::ConditionalFilter do
+  describe '#configure' do
+
+    context "success" do
+      let(:conf) {
+        %[
+          key_pattern @example\.com$
+          condition   10
+          filter      numeric_upward
+        ]
+      }
+
+      let(:driver) { Fluent::Test::Driver::Filter.new(described_class).configure(conf) }
+      subject {
+        driver
+      }
+
+      it {
+        expect(subject.instance).to be_an_instance_of described_class
+        expect(subject.instance.key_pattern).to be == "@example\.com$"
+        expect(subject.instance.instance_variable_get(:@key_pattern_regexp)).to be == /@example.com$/
+      }
+    end
+
+    context "failure" do
+      context 'key_pattern not set' do
+        let(:conf) {
+          %[
+            condition   10
+            filter      numeric_upward
+          ]
+        }
+
+        it {
+          expect {
+            Fluent::Test::Driver::Filter.new(described_class).configure(conf)
+          }.to raise_error(Fluent::ConfigError)
+        }
+      end
+
+      context 'condition not set' do
+        let(:conf) {
+          %[
+            key_pattern @example.com$
+            filter      numeric_upward
+          ]
+        }
+
+        it {
+          expect {
+            Fluent::Test::Driver::Filter.new(described_class).configure(conf)
+          }.to raise_error(Fluent::ConfigError)
+        }
+      end
+
+      context 'filter not set' do
+        let(:conf) {
+          %[
+            key_pattern @example.com$
+            condition   10
+          ]
+        }
+
+        it {
+          expect {
+            Fluent::Test::Driver::Filter.new(described_class).configure(conf)
+          }.to raise_error(Fluent::ConfigError)
+        }
+      end
+    end
+  end
+
+  describe "#filter" do
+    context('numeric_upward') do
+      let(:conf) {
+        %[
+          key_pattern @example.com$
+          condition   10
+          filter      numeric_upward
+        ]
+      }
+
+      let(:driver) { Fluent::Test::Driver::Filter.new(described_class).configure(conf) }
+
+      context('with 0 matched key/value pair') do
+        before {
+          driver.run(default_tag: 'test') {
+            driver.feed('foo@example.com' => 8, 'bar@example.com' => 6, 'baz@baz.com' => 15)
+          }
+        }
+
+        it {
+          expect(driver.filtered[0]).to be_nil
+        }
+      end
+
+      context('with 1 matched key/value pair') do
+        before {
+          driver.run(default_tag: 'test') {
+            driver.feed('foo@example.com' => 12, 'bar@example.com' => 6, 'baz@baz.com' => 15)
+          }
+        }
+
+        it {
+          expect(driver.filtered[0][1].keys.length).to be == 1
+        }
+      end
+
+      context('with 2 matched key/value pairs') do
+        before {
+          driver.run(default_tag: 'test') {
+            driver.feed('foo@example.com' => 12, 'bar@example.com' => 10, 'baz@baz.com' => 15)
+          }
+        }
+
+        it {
+          expect(driver.filtered[0][1].keys.length).to be == 2
+        }
+      end
+    end
+
+    context('numeric_downward') do
+      let(:conf) {
+        %[
+          key_pattern @example.com$
+          condition   10
+          filter      numeric_downward
+        ]
+      }
+
+      let(:driver) { Fluent::Test::Driver::Filter.new(described_class).configure(conf) }
+
+      context('with 0 matched key/value pair') do
+        before {
+          driver.run(default_tag: 'test') {
+            driver.feed('foo@example.com' => 18, 'bar@example.com' => 26, 'baz@baz.com' => 15)
+          }
+        }
+
+        it {
+          expect(driver.filtered[0]).to be_nil
+        }
+      end
+
+      context('with 1 matched key/value pair') do
+        before {
+          driver.run(default_tag: 'test') {
+            driver.feed('foo@example.com' => 11, 'bar@example.com' => 6, 'baz@baz.com' => 5)
+          }
+        }
+
+        it {
+          expect(driver.filtered[0][1].keys.length).to be == 1
+        }
+      end
+
+      context('with 2 matched key/value pairs') do
+        before {
+          driver.run(default_tag: 'test') {
+            driver.feed('foo@example.com' => 10, 'bar@example.com' => 5, 'baz@baz.com' => 5)
+          }
+        }
+
+        it {
+          expect(driver.filtered[0][1].keys.length).to be == 2
+        }
+      end
+    end
+
+    context('string_match') do
+      let(:conf) {
+        %[
+          key_pattern @example.com$
+          condition   (staff|user)
+          filter      string_match
+        ]
+      }
+
+      let(:driver) { Fluent::Test::Driver::Filter.new(described_class).configure(conf) }
+
+      context('with 0 matched key/value pair') do
+        before {
+          driver.run(default_tag: 'test') {
+            driver.feed('foo@example.com' => 'guest', 'bar@example.com' => 'guest', 'baz@baz.com' => 'staff')
+          }
+        }
+
+        it {
+          expect(driver.filtered[0]).to be_nil
+        }
+      end
+
+      context('with 1 matched key/value pair') do
+        before {
+          driver.run(default_tag: 'test') {
+            driver.feed('foo@example.com' => 'staff', 'bar@example.com' => 'guest', 'baz@baz.com' => 'user')
+          }
+        }
+
+        it {
+          expect(driver.filtered[0][1].keys.length).to be == 1
+        }
+      end
+
+      context('with 2 matched key/value pairs') do
+        before {
+          driver.run(default_tag: 'test') {
+            driver.feed('foo@example.com' => 'staff', 'bar@example.com' => 'user', 'baz@baz.com' => 'staff')
+          }
+        }
+
+        it {
+          expect(driver.filtered[0][1].keys.length).to be == 2
+        }
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,5 +1,7 @@
 require 'fluent/test'
+require 'fluent/test/driver/filter'
 require 'fluent/plugin/out_conditional_filter'
+require 'fluent/plugin/filter_conditional'
 
 RSpec.configure do |config|
   config.before(:all) do
@@ -24,4 +26,3 @@ unless ENV.has_key?('VERBOSE')
   }
   $log = nulllogger
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-conditional_filter
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Kentaro Kuribayashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-04 00:00:00.000000000 Z
+date: 2017-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -94,7 +100,10 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-conditional_filter.gemspec
+- lib/fluent/plugin/conditional_filter_rule.rb
+- lib/fluent/plugin/filter_conditional.rb
 - lib/fluent/plugin/out_conditional_filter.rb
+- spec/lib/filter_conditional_spec.rb
 - spec/lib/out_conditional_filter_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/kentaro/fluent-plugin-conditional_filter
@@ -122,5 +131,6 @@ signing_key:
 specification_version: 4
 summary: A fluent plugin that provides conditional filters
 test_files:
+- spec/lib/filter_conditional_spec.rb
 - spec/lib/out_conditional_filter_spec.rb
 - spec/spec_helper.rb