fluentd 0.12.1 → 0.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec182ccd57b8d91c6b246b25783b1b34487ad43f
-  data.tar.gz: 3194c1994b373e087aabb571b2b5038f446740d7
+  metadata.gz: b7788505e84495465d06f359b75401362f04caaa
+  data.tar.gz: 47a5ed1ef3ee3b55cabbcfdf65877374e66642b7
 SHA512:
-  metadata.gz: 8edee638e09a09b8246bb36e85f9111490f39826af203fe9669a49c0ec6493ff6b4b4fb3cbbbb6b4b4430a3483439af04663e6eb4e2be9897ece3f2479ed10e7
-  data.tar.gz: 25d045c021631970ddb83499601c3b569efab22b8abc4b893a3f62e06712d3921b7a808a1fb740abd087516d6c0d3c8eb106ccf1b7364a057b4954ae34eaec85
+  metadata.gz: 7b174bf87334d6c06db915f3c244e39dbc637c7f21872fc227046f9b8c4e332809972e956013c0f0c4fea1ed187e89a45656723d698e733f8fcf78e85cf8f7d0
+  data.tar.gz: 602154651def0f60ddc693b6ba1b1bd5c7d7ce0f2efbc7b4413ba16905929239fb7928e41fd00f93838f4a3ee909f5978737dded60b2fcf6c8fd035221cc381d

data/ChangeLog

@@ -2,6 +2,21 @@
 
 ## Release 0.12.1 - 2014/12/14
 
+### New Features
+
+* filter: Change filter behaviour. If filter method returns nil, its record is removed from stream.
+
+### Enhancement
+
+* in_monitor_agent: Render filter plugins
+* test: FilterTestDriver has run, emit and getting result methods
+
+### Bug fixes
+
+* test: Support @label by TestDriver
+
+## Release 0.12.1 - 2014/12/14
+
 ### Bug fixes
 
 * plugin: Fix router is nil inside out_copy, out_roundrobin, and BufferedOutput secondary

data/lib/fluent/filter.rb

@@ -44,13 +44,15 @@ module Fluent
     end
 
     def filter(tag, time, record)
+      raise NotImplementedError, "Implement this method in child class"
     end
 
     def filter_stream(tag, es)
       new_es = MultiEventStream.new
       es.each { |time, record|
         begin
-          new_es.add(time, filter(tag, time, record))
+          filtered_record = filter(tag, time, record)
+          new_es.add(time, filtered_record) if filtered_record
         rescue => e
           router.emit_error_event(tag, time, record, e)
         end

data/lib/fluent/plugin/filter_grep.rb

@@ -49,9 +49,9 @@ module Fluent
       end
     end
 
-    def filter_stream(tag, es)
-      result_es = MultiEventStream.new
-      es.each do |time, record|
+    def filter(tag, time, record)
+      result = nil
+      begin
         catch(:break_loop) do
           @regexps.each do |key, regexp|
             throw :break_loop unless match(regexp, record[key].to_s)
@@ -59,13 +59,13 @@ module Fluent
           @excludes.each do |key, exclude|
             throw :break_loop if match(exclude, record[key].to_s)
           end
-          result_es.add(time, record)
+          result = record
         end
+      rescue => e
+        log.warn "failed to grep events", :error_class => e.class, :error => e.message
+        log.warn_backtrace
       end
-      result_es
-    rescue => e
-      log.warn "failed to grep events", :error_class => e.class, :error => e.message
-      log.warn_backtrace
+      result
     end
 
     private

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -228,7 +228,7 @@ module Fluent
     end
 
     MONITOR_INFO = {
-      'output_plugin' => 'is_a?(::Fluent::Output)',
+      'output_plugin' => 'is_a?(::Fluent::Output)', # deprecated. Use plugin_category instead
       'buffer_queue_length' => '@buffer.queue_size',
       'buffer_total_queued_size' => '@buffer.total_queued_chunk_size',
       'retry_count' => '@num_errors',
@@ -244,11 +244,14 @@ module Fluent
       Engine.root_agent.outputs.each { |o|
         MonitorAgentInput.collect_children(o, array)
       }
+      # get all filter plugins
+      Engine.root_agent.filters.each { |f|
+        MonitorAgentInput.collect_children(f, array)
+      }
       Engine.root_agent.labels.each { |name, l|
-        # TODO: Add label name to outputs for identifing plugins
-        l.outputs.each { |o|
-          MonitorAgentInput.collect_children(o, array)
-        }
+        # TODO: Add label name to outputs / filters for identifing plugins
+        l.outputs.each { |o| MonitorAgentInput.collect_children(o, array) }
+        l.filters.each { |f| MonitorAgentInput.collect_children(f, array) }
       }
 
       array
@@ -348,6 +351,8 @@ module Fluent
         'input'.freeze
       when Fluent::Output
         'output'.freeze
+      when Fluent::Filter
+        'filter'.freeze
       else
         'unknown'.freeze
       end

data/lib/fluent/test/base.rb

@@ -59,15 +59,19 @@ module Fluent
         else
           @config = Config.parse(str, "(test)", "(test_dir)", use_v1)
         end
+        if label_name = @config['@label']
+          Engine.root_agent.add_label(label_name)
+        end
         @instance.configure(@config)
         self
       end
 
-      def run(&block)
+      # num_waits is for checking thread status. This will be removed after improved plugin API
+      def run(num_waits = 10, &block)
         @instance.start
         begin
           # wait until thread starts
-          10.times { sleep 0.05 }
+          num_waits.times { sleep 0.05 }
           return yield
         ensure
           @instance.shutdown

data/lib/fluent/test/filter_test.rb

@@ -17,16 +17,57 @@
 module Fluent
   module Test
     class FilterTestDriver < TestDriver
-      def initialize(klass, &block)
+      def initialize(klass, tag = 'filter.test', &block)
         super(klass, &block)
+        @tag = tag
+        @events = {}
+        @filtered = MultiEventStream.new
       end
 
-      def filter(tag, time, record)
-        @instance.filter(tag, time, record)
+      attr_reader :filtered
+      attr_accessor :tag
+
+      def emit(record, time = Engine.now)
+        emit_with_tag(@tag, record, time)
+      end
+      alias_method :filter, :emit
+
+      def emit_with_tag(tag, record, time = Engine.now)
+        @events[tag] ||= MultiEventStream.new
+        @events[tag].add(time, record)
+      end
+      alias_method :filter_with_tag, :emit_with_tag
+
+      def filter_stream(es)
+        filter_stream_with_tag(@tag, es)
       end
 
-      def filter_stream(tag, es)
-        @instance.filter_stream(tag, es)
+      def filter_stream_with_tag(tag, es)
+        @events[tag] = es
+      end
+
+      def filtered_as_array
+        all = []
+        @filtered.each { |time, record|
+          all << [@tag, time, record]
+        }
+        all
+      end
+      alias_method :emits, :filtered_as_array # emits is for consistent with other drivers
+
+      # Almost filters don't use threads so default is 0. It reduces test time.
+      def run(num_waits = 0, &block)
+        super(num_waits) {
+          block.call if block
+
+          @events.each { |tag, es|
+            processed = @instance.filter_stream(tag, es)
+            processed.each { |time, record|
+              @filtered.add(time, record)
+            }
+          }
+        }
+        self
       end
     end
   end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.1'
+  VERSION = '0.12.2'
 
 end

data/test/plugin/test_filter_grep.rb

@@ -42,13 +42,12 @@ class GrepFilterTest < Test::Unit::TestCase
     end
 
     def emit(config, msgs)
-      es = Fluent::MultiEventStream.new
-      msgs.each { |msg|
-        es.add(@time, {'foo' => 'bar', 'message' => msg})
-      }
-
       d = create_driver(config)
-      d.filter_stream('filter.test', es);
+      d.run {
+        msgs.each { |msg|
+          d.emit({'foo' => 'bar', 'message' => msg}, @time)
+        }
+      }.filtered
     end
 
     test 'empty config' do
@@ -93,11 +92,9 @@ class GrepFilterTest < Test::Unit::TestCase
 
   sub_test_case 'grep non-string jsonable values' do
     def emit(msg, config = 'regexp1 message 0')
-      es = Fluent::MultiEventStream.new
-      es.add(@time, {'foo' => 'bar', 'message' => msg})
-
       d = create_driver(config)
-      d.filter_stream('filter.test', es);
+      d.emit({'foo' => 'bar', 'message' => msg}, @time)
+      d.run.filtered
     end
 
     data(

data/test/plugin/test_filter_record_transformer.rb

@@ -19,7 +19,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
   end
 
   def create_driver(conf = '')
-    Test::FilterTestDriver.new(RecordTransformerFilter).configure(conf, true)
+    Test::FilterTestDriver.new(RecordTransformerFilter, @tag).configure(conf, true)
   end
 
   sub_test_case 'configure' do
@@ -38,13 +38,12 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
 
   sub_test_case "test options" do
     def emit(config, msgs = [''])
-      es = Fluent::MultiEventStream.new
-      msgs.each do |msg|
-        es.add(@time, {'foo' => 'bar', 'message' => msg})
-      end
-
       d = create_driver(config)
-      d.filter_stream(@tag, es)
+      d.run {
+        msgs.each { |msg|
+          d.emit({'foo' => 'bar', 'message' => msg}, @time)
+        }
+      }.filtered
     end
 
     CONFIG = %[
@@ -120,14 +119,13 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
 
   sub_test_case 'test placeholders' do
     def emit(config, msgs = [''])
-      es = Fluent::MultiEventStream.new
-      msgs.each do |msg|
-        es.add(@time, {'eventType0' => 'bar', 'message' => msg})
-      end
-
       d = create_driver(config)
       yield d if block_given?
-      d.filter_stream(@tag, es)
+      d.run {
+        msgs.each do |msg|
+          d.emit({'eventType0' => 'bar', 'message' => msg}, @time)
+        end
+      }.filtered
     end
 
     %w[yes no].each do |enable_ruby|

data/test/test_filter.rb

@@ -0,0 +1,79 @@
+require_relative 'helper'
+require 'fluent/filter'
+
+class FilterTest < Test::Unit::TestCase
+  include Fluent
+
+  setup do
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  def create_driver(klass = Fluent::Filter, conf = '')
+    Test::FilterTestDriver.new(klass).configure(conf, true)
+  end
+
+  def emit(klass, msgs, conf = '')
+    d = create_driver(klass, conf)
+    d.run {
+      msgs.each {|msg|
+        d.emit({'message' => msg}, @time)
+      }
+    }.filtered
+  end
+
+  sub_test_case 'configure' do
+    test 'check default' do
+      assert_nothing_raised { create_driver }
+    end
+  end
+
+  sub_test_case 'filter' do
+    test 'NotImplementedError' do
+      not_implemented_filter = Class.new(Fluent::Filter)
+      assert_raise(NotImplementedError) { emit(not_implemented_filter, ['foo']) }
+    end
+
+    test 'null filter' do
+      null_filter = Class.new(Fluent::Filter) do |c|
+        def filter(tag, time, record)
+          nil
+        end
+      end
+      es = emit(null_filter, ['foo'])
+      assert_equal(0, es.instance_variable_get(:@record_array).size)
+    end
+
+    test 'pass filter' do
+      pass_filter = Class.new(Fluent::Filter) do |c|
+        def filter(tag, time, record)
+          record
+        end
+      end
+      es = emit(pass_filter, ['foo'])
+      assert_equal(1, es.instance_variable_get(:@record_array).size)
+    end
+  end
+
+  sub_test_case 'filter_stream' do
+    test 'null filter' do
+      null_filter = Class.new(Fluent::Filter) do |c|
+        def filter_stream(tag, es)
+          MultiEventStream.new
+        end
+      end
+      es = emit(null_filter, ['foo'])
+      assert_equal(0, es.instance_variable_get(:@record_array).size)
+    end
+
+    test 'pass filter' do
+      pass_filter = Class.new(Fluent::Filter) do |c|
+        def filter_stream(tag, es)
+          es
+        end
+      end
+      es = emit(pass_filter, ['foo'])
+      assert_equal(1, es.instance_variable_get(:@record_array).size)
+    end
+  end
+end

data/test/test_input.rb

@@ -12,19 +12,20 @@ class FluentInputTest < ::Test::Unit::TestCase
     Fluent::Test::InputTestDriver.new(Fluent::Input).configure(conf, true)
   end
 
-  def test_configure
+  def test_router
     d = create_driver
     assert_equal Engine.root_agent.event_router, d.instance.router
 
-    assert_raise(ArgumentError) {
-      create_driver('@label @unknown')
-    }
-
-    Engine.root_agent.add_label('@known')
     d = nil
     assert_nothing_raised { 
       d = create_driver('@label @known')
     }
-    assert d.instance.router
+    expected = Engine.root_agent.find_label('@known').event_router
+    assert_equal expected, d.instance.router
+
+    # TestDriver helps to create a label instance automatically, so directly test here
+    assert_raise(ArgumentError) {
+      Fluent::Input.new.configure(Config.parse('@label @unknown', '(test)', '(test_dir)', true))
+    }
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.12.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-14 00:00:00.000000000 Z
+date: 2014-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -426,6 +426,7 @@ files:
 - test/test_config.rb
 - test/test_configdsl.rb
 - test/test_event_router.rb
+- test/test_filter.rb
 - test/test_formatter.rb
 - test/test_input.rb
 - test/test_match.rb
@@ -506,6 +507,7 @@ test_files:
 - test/test_config.rb
 - test/test_configdsl.rb
 - test/test_event_router.rb
+- test/test_filter.rb
 - test/test_formatter.rb
 - test/test_input.rb
 - test/test_match.rb