fluent-plugin-eval-filter 0.0.5 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d3319b1bf8027598860812fbd405a53ea82e731
-  data.tar.gz: d259849f8c620147be63c03e0a1ba564a3efd83c
+  metadata.gz: e111ef217c550c8411a41a3d3f8955d0b101211c
+  data.tar.gz: 030da12335bab1ba7fe71c6287891c03127a90e1
 SHA512:
-  metadata.gz: ff93e5e059ef088710c0799bda409c47a715477e1952c2afa942f92cd0db1db433c539804fe69cbaedb2002d6ecf8a1e0f31f565e29d7356c1c08e136ab611b5
-  data.tar.gz: 672cd6cda516a80afae3ff21bc1124030df5179171b8faecb6c8a9e996cbcef38baf84347014777efa562a8fafe5c0bf5d0db06b6f46459fe5042225ab8c4af5
+  metadata.gz: 420ab83d2f9be13c25f172210c98ebdc830bad7b2550e47ba7a68225011340d1247f4404a580f484cdeaa54572d656a9762e3fd30cd47da17fda334f59d7820f
+  data.tar.gz: e42b7f259334fc99085d6cc36a1d06a3e9f13773af31cbadf8b67efd090aaa0cd01b3671f2d27f238f988c8b66c0d634536e06db635a791c30195efd28b5e36e

data/.travis.yml

@@ -1,8 +1,9 @@
 language: ruby
 rvm:
+  - 2.3.0
+  - 2.2.0
   - 2.1.0
   - 2.0.0
   - 1.9.3
-  - 1.9.2
 before_install:
   - gem update --remote bundler

data/README.md

@@ -28,6 +28,67 @@ Or install it yourself as:
 </match>
 ```
 
+### require libraries
+```
+<match raw.apache.access>
+  type eval_filter
+  remove_tag_prefix raw
+  add_tag_prefix filtered
+  requires yaml # comma separated values
+
+  config1 @hostname = YAML.load({'hostname' => 'web01'})['hostname']
+
+  filter1 [[tag, @hostname].join('.'), time, record] if record['method'] == 'GET'
+</match>
+```
+
+## Filter Plugin
+
+Note that this filter version does not have rewrite tag functionality.
+Should return [time, record].
+
+## Configuration
+
+
+### filter:
+
+    <filter **>
+      type eval
+      filter1 "[time, record] if record['status'] == '404'"
+      filter2 "[time, record] if record['status'] == 'POST'"
+    </filter>
+
+
+### typecast(string to integer):
+
+    <filter **>
+      type eval
+      filter1 "record['status'] = record['status'].to_i; [time, record]"
+    </filter>
+
+### modify record(add value):
+
+    <filter **>
+      type eval
+      filter1 "record['user_id'] = record['message'].split(':').last.to_i; [time, record]"
+    </filter>
+
+#### input
+    {'status' => '301', 'message' => 'user_id:1'}
+    {'status' => '302', 'message' => 'user_id:2'}
+    {'status' => '404', 'message' => 'user_id:3'}
+    {'status' => '503', 'message' => 'user_id:4'}
+    {'status' => '401', 'message' => 'user_id:5'}
+
+#### output
+    {'status' => '301', 'message' => 'user_id:1', 'user_id' => 1}
+    {'status' => '302', 'message' => 'user_id:2', 'user_id' => 2}
+    {'status' => '404', 'message' => 'user_id:3', 'user_id' => 3}
+    {'status' => '503', 'message' => 'user_id:4', 'user_id' => 4}
+    {'status' => '401', 'message' => 'user_id:5', 'user_id' => 5}
+
+
+
 ## Limitation
 
 Can not be used expression substitution.

data/fluent-plugin-eval-filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-eval-filter"
-  spec.version       = "0.0.5"
+  spec.version       = "0.1.0"
   spec.authors       = ["Yuzuki Masaru"]
   spec.email         = ["ephemeralsnow@gmail.com"]
   spec.description   = %q{Fluentd Output eval filter plugin.}
@@ -19,4 +19,5 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "fluentd", "~> 0"
   spec.add_development_dependency "rake", "~> 0"
+  spec.add_development_dependency "test-unit", "~> 3.1.0"
 end

data/lib/fluent/plugin/filter_eval.rb

@@ -0,0 +1,68 @@
+module Fluent
+  class EvalFilter < Filter
+    Fluent::Plugin.register_filter('eval', self)
+
+    config_param :requires, :string, default: nil, :desc => "require libraries."
+
+    def initialize
+      super
+    end
+
+    def configure(conf)
+      super
+
+      if @requires
+        @requires.split(',').each do |lib|
+          begin
+            require lib
+          rescue Exception => e
+            raise Fluent::ConfigError, "\n#{e.message}\n#{e.backtrace.join("\n")}"
+          end
+        end
+      end
+
+      conf.keys.select { |key| key =~ /^config\d+$/ }.sort_by { |key| key.sub('config', '').to_i }.each do |key|
+        begin
+          instance_eval("#{conf[key]}")
+        rescue Exception => e
+          raise Fluent::ConfigError, "#{key} #{conf[key]}\n" + e.to_s
+        end
+      end
+
+      @filters = []
+      conf.keys.select { |key| key =~ /^filter\d+$/ }.sort_by { |key| key.sub('filter', '').to_i }.each do |key|
+        begin
+          @filters << instance_eval("lambda do |tag, time, record| #{conf[key]} end")
+        rescue Exception => e
+          raise Fluent::ConfigError, "#{key} #{conf[key]}\n" + e.to_s
+        end
+      end
+
+      if @filters.empty?
+        raise Fluent::ConfigError, "missing filters"
+      end
+    end
+
+    def filter_stream(tag, es)
+      new_es = MultiEventStream.new
+      es.each { |time, record|
+        begin
+          filtered_record = filter_record(tag, time, record)
+          new_es.add(*filtered_record) if filtered_record
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
+      }
+      new_es
+    end
+
+    private
+    def filter_record(tag, time, record)
+      @filters.each do |filter|
+        filter_results = filter.call(tag, time, record)
+        return filter_results if filter_results
+      end
+      nil
+    end
+  end
+end

data/lib/fluent/plugin/out_eval_filter.rb

@@ -2,9 +2,26 @@ class Fluent::EvalFilterOutput < Fluent::Output
 
   Fluent::Plugin.register_output('eval_filter', self)
 
+  config_param :requires, :string, default: nil, :desc => "require libraries."
+
+  # Define `router` method of v0.12 to support v0.10 or earlier
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
   def configure(conf)
     super
 
+    if @requires
+      @requires.split(',').each do |lib|
+        begin
+          require lib
+        rescue Exception => e
+          raise Fluent::ConfigError, "\n#{e.message}\n#{e.backtrace.join("\n")}"
+        end
+      end
+    end
+
     if remove_tag_prefix = conf['remove_tag_prefix']
       @remove_tag_prefix = /^#{Regexp.escape(remove_tag_prefix)}\.*/
     end
@@ -42,7 +59,7 @@ class Fluent::EvalFilterOutput < Fluent::Output
       results = filter_record(tag, time, record)
       if results
         results.each do |result|
-          Fluent::Engine.emit(*result)
+          router.emit(*result)
         end
       end
     end

data/test/helper.rb

@@ -23,6 +23,7 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/out_eval_filter'
+require 'fluent/plugin/filter_eval'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_filter_eval.rb

@@ -0,0 +1,128 @@
+require 'helper'
+
+class EvalFilterTest < Test::Unit::TestCase
+  include Fluent
+
+  setup do
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  def create_driver(conf = '')
+    Test::FilterTestDriver.new(EvalFilter).configure(conf)
+  end
+
+  def filter(config, msgs)
+    d = create_driver(config)
+    d.run {
+      msgs.each {|msg|
+        d.filter(msg, @time) # Filterプラグインにメッセージを通す
+      }
+    }
+    filtered = d.filtered_as_array # 結果を受け取る. [tag, time, record]の配列
+    filtered
+  end
+
+  sub_test_case 'configure' do
+    test 'check default' do
+      config = %[
+        filter1 [time, record] if record['status'] == '404'
+      ]
+      assert_nothing_raised {
+        create_driver(config)
+      }
+
+      assert_raise(Fluent::ConfigError) do
+        create_driver('')
+      end
+    end
+  end
+
+  sub_test_case 'filter stream' do
+    test 'filter' do
+      msgs = [
+        {'status' => '301', 'message' => 'message'},
+        {'status' => '302', 'message' => 'message'},
+        {'status' => '404', 'message' => 'message'},
+        {'status' => '503', 'message' => 'message'},
+        {'status' => '401', 'message' => 'message'}
+      ]
+      config = %[
+        filter1 [time, record] if record['status'] == '404'
+        filter2 [time, record] if record['status'] == '503'
+      ]
+      es = filter(config, msgs)
+      assert_equal(es.size, 2)
+      assert_equal(es[0][2]['status'], '404')
+      assert_equal(es[1][2]['status'], '503')
+    end
+  end
+
+  sub_test_case 'convert type' do
+    test 'to_i' do
+      msgs = [
+        {'status' => '301', 'message' => 'message'},
+        {'status' => '302', 'message' => 'message'},
+        {'status' => '404', 'message' => 'message'},
+        {'status' => '503', 'message' => 'message'},
+        {'status' => '401', 'message' => 'message'}
+      ]
+      config = %[
+        filter1 record['status'] = record['status'].to_i; [time, record]
+      ]
+      es = filter(config, msgs)
+      assert_equal(es.size, 5)
+      assert_equal(es[0][2]['status'], 301)
+      assert_equal(es[1][2]['status'], 302)
+      assert_equal(es[2][2]['status'], 404)
+      assert_equal(es[3][2]['status'], 503)
+      assert_equal(es[4][2]['status'], 401)
+    end
+  end
+
+  sub_test_case 'add value' do
+    test 'add user_id' do
+      msgs = [
+        {'status' => '301', 'message' => 'user_id:1'},
+        {'status' => '302', 'message' => 'user_id:2'},
+        {'status' => '404', 'message' => 'user_id:3'},
+        {'status' => '503', 'message' => 'user_id:4'},
+        {'status' => '401', 'message' => 'user_id:5'}
+      ]
+      config = %[
+        filter1 record['user_id'] = record['message'].split(':').last.to_i; [time, record]
+      ]
+      es = filter(config, msgs)
+      assert_equal(es.size, 5)
+      assert_equal(es[0][2]['user_id'], 1)
+      assert_equal(es[1][2]['user_id'], 2)
+      assert_equal(es[2][2]['user_id'], 3)
+      assert_equal(es[3][2]['user_id'], 4)
+      assert_equal(es[4][2]['user_id'], 5)
+    end
+  end
+
+  sub_test_case 'require libraries' do
+    test 'require yaml' do
+      config = %[
+        requires yaml
+        filter1 record.to_yaml; [time, record]
+      ]
+      assert_nothing_raised {
+        create_driver(config)
+        es = filter(config, [{'key' => 'value'}])
+        assert_equal(es.size, 1)
+      }
+    end
+
+    test 'require error' do
+      config = %[
+        requires hoge
+        filter1 record.to_yaml; [time, record]
+      ]
+      assert_raise(Fluent::ConfigError) do
+        create_driver(config)
+      end
+    end
+  end
+end

data/test/plugin/test_out_eval_filter.rb

@@ -1,6 +1,6 @@
 require 'helper'
 
-class EvalFilterTest < Test::Unit::TestCase
+class EvalFilterOutputTest < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
@@ -296,4 +296,23 @@ class EvalFilterTest < Test::Unit::TestCase
     assert_equal 'test3', emits[2][2]['value']
   end
 
+  def test_require_libraries
+    d = create_driver(%[
+      requires yaml
+      filter1 record.to_yaml; ['tag', 0, record]
+    ])
+    assert_nothing_raised {
+      d.run { d.emit({'key' => 'value'}) }
+    }
+  end
+
+  def test_require_error
+    assert_raise(Fluent::ConfigError) do
+      d = create_driver(%[
+        requires hoge
+        filter1 record.to_yaml; ['tag', 0, record]
+      ])
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-eval-filter
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
 platform: ruby
 authors:
 - Yuzuki Masaru
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-25 00:00:00.000000000 Z
+date: 2016-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
 description: Fluentd Output eval filter plugin.
 email:
 - ephemeralsnow@gmail.com
@@ -52,8 +66,10 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-eval-filter.gemspec
+- lib/fluent/plugin/filter_eval.rb
 - lib/fluent/plugin/out_eval_filter.rb
 - test/helper.rb
+- test/plugin/test_filter_eval.rb
 - test/plugin/test_out_eval_filter.rb
 homepage: https://github.com/ephemeralsnow/fluent-plugin-eval-filter
 licenses:
@@ -75,10 +91,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Fluentd Output eval filter plugin.
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_eval.rb
 - test/plugin/test_out_eval_filter.rb