fluent-plugin-route 0.2.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7740164b77dfc4121d2abb21a984958cc4578cc8
-  data.tar.gz: 22a5547cf3d120b4e939b232b858f9482935f2f0
+  metadata.gz: a20d74801e0b2189c8de4e61be4123e131986800
+  data.tar.gz: 68d0a470e298074ce667651a6f4acccf79738d09
 SHA512:
-  metadata.gz: 990730d0d8956c835ccb2e9b7e1e88e9d7ad0df1feb88f1736db30279d371ad42908eb9b349ddd64f0e019d7a3242dedc4120091daa0b03aacc20302378c6c93
-  data.tar.gz: f840ec3d77b45fe782871621793ef2ff4401c3dddcbce08107072aa808f2537e5c9b207745ac28e0ef8ad240c293e33a0fb08fd3dec2545bd6dae105366d54be
+  metadata.gz: 64ad0ac9468fb83c7c6c7e0a35c2e02b9325c5c26ccd7ddff6f86ef71e644e2657b0530003b82d0023b288ff1d51f0a36313544c3bddc3eeac3179389460160d
+  data.tar.gz: fb4e5e8fa31fdce80f2a4e577c392d734fe04c59d3e268600139044e991ef8456f33780760f4a858e1a9cbdf695f9be176bdb644e3c22df4e9aa930ef70447af

data/fluent-plugin-route.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-route"
-  gem.version       = "0.2.0"
+  gem.version       = "1.0.0"
   gem.authors       = ["TAGOMORI Satoshi", "FURUHASHI Sadayuki"]
   gem.email         = ["tagomoris@gmail.com", "frsyuki@gmail.com"]
   gem.summary       = %q{Fluentd plugin to route messages in fluentd processes}
@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_runtime_dependency "fluentd", "< 0.14.0"
+  gem.add_runtime_dependency "fluentd", ">= 0.14.0"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"
 end

data/lib/fluent/plugin/out_route.rb

@@ -15,153 +15,131 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-require 'fluent/output'
+require 'fluent/plugin/bare_output'
+require 'fluent/match'
 
-module Fluent
-  class RouteOutput < MultiOutput
-    Plugin.register_output('route', self)
+module Fluent::Plugin
+  class RouteOutput < BareOutput
+    Fluent::Plugin.register_output('route', self)
 
-    class Route
-      include Configurable
+    helpers :event_emitter
 
-      config_param :remove_tag_prefix, :string, :default => nil
-      config_param :add_tag_prefix, :string, :default => nil
-      # TODO tag_transform regexp
-      attr_accessor :copy
+    config_section :route, param_name: :route_configs, multi: true, required: true do
+      config_argument :pattern, :string, default: '**'
+      config_param :@label, :string, default: nil
+      config_param :remove_tag_prefix, :string, default: nil
+      config_param :add_tag_prefix, :string, default: nil
+      config_param :copy, :bool, default: false
+    end
 
-      def initialize(pattern, router)
-        super()
-        if !pattern || pattern.empty?
-          pattern = '**'
-        end
-        @router = router
-        @pattern = MatchPattern.create(pattern)
-        @tag_cache = {}
-      end
+    config_param :remove_tag_prefix, :string, default: nil
+    config_param :add_tag_prefix, :string, default: nil
 
-      def match?(tag)
-        @pattern.match(tag)
-      end
+    config_param :match_cache_size, :integer, default: 256
+    config_param :tag_cache_size, :integer, default: 256
 
-      def configure(conf)
-        super
-        if conf['copy']
-          @copy = true
-        else
-          @copy = false
-        end
-        if label_name = conf['@label']
-          label = Fluent::Engine.root_agent.find_label(label_name)
-          @router = label.event_router
-        end
-        if @remove_tag_prefix
-          @prefix_match = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
-        else
-          @prefix_match = //
-        end
-        if @add_tag_prefix
-          @tag_prefix = "#{@add_tag_prefix}."
-        else
-          @tag_prefix = ""
-        end
-      end
-
-      def copy?
-        @copy
-      end
+    attr_reader :routes
 
-      def emit(tag, es)
-        ntag = @tag_cache[tag]
-        unless ntag
-          ntag = tag.sub(@prefix_match, @tag_prefix)
-          if @tag_cache.size < 1024  # TODO size limit
-            @tag_cache[tag] = ntag
+    def tag_modifier(remove_tag_prefix, add_tag_prefix)
+      tag_cache_size = @tag_cache_size
+      cache = {}
+      mutex = Mutex.new
+      removed_prefix = remove_tag_prefix ? remove_tag_prefix + "." : ""
+      added_prefix = add_tag_prefix ? add_tag_prefix + "." : ""
+      ->(tag){
+        if cached = cache[tag]
+          cached
+        else
+          modified = tag.start_with?(removed_prefix) ? tag.sub(removed_prefix, added_prefix) : added_prefix + tag
+          mutex.synchronize do
+            if cache.size >= tag_cache_size
+              remove_keys = cache.keys[0...(tag_cache_size / 2)]
+              cache.delete_if{|key, _value| remove_keys.include?(key) }
+            end
+            cache[tag] = modified
           end
+          modified
         end
-        @router.emit_stream(ntag, es)
-      end
-    end
-
-    def initialize
-      super
-      @routes = []
-      @tag_cache = {}
-      @match_cache = {}
+      }
     end
 
-    config_param :remove_tag_prefix, :string, :default => nil
-    config_param :add_tag_prefix, :string, :default => nil
-    # TODO tag_transform regexp
-
-    attr_reader :routes
+    def configure(conf)
+      if conf.elements(name: 'store').size > 0
+        raise Fluent::ConfigError, "<store> section is not available in route plugin"
+      end
 
-    # Define `log` method for v0.10.42 or earlier
-    unless method_defined?(:log)
-      define_method("log") { $log }
-    end
+      super
 
-    # Define `router` method of v0.12 to support v0.10 or earlier
-    unless method_defined?(:router)
-      define_method("router") { ::Fluent::Engine }
+      @match_cache = {}
+      @routes = []
+      @route_configs.each do |rc|
+        route_router = event_emitter_router(rc['@label'])
+        modifier = tag_modifier(rc.remove_tag_prefix, rc.add_tag_prefix)
+        @routes << Route.new(rc.pattern, route_router, modifier, rc.copy)
+      end
+      @default_tag_modifier = (@remove_tag_prefix || @add_tag_prefix) ? tag_modifier(@remove_tag_prefix, @add_tag_prefix) : nil
+      @mutex = Mutex.new
     end
 
-    def configure(conf)
-      super
+    class Route
+      def initialize(pattern, router, tag_modifier, copy)
+        @router = router
+        @pattern = Fluent::MatchPattern.create(pattern)
+        @tag_modifier = tag_modifier
+        @copy = copy
+      end
 
-      if @remove_tag_prefix
-        @prefix_match = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
-      else
-        @prefix_match = //
+      def match?(tag)
+        @pattern.match(tag)
       end
-      if @add_tag_prefix
-        @tag_prefix = "#{@add_tag_prefix}."
-      else
-        @tag_prefix = ""
+
+      def copy?
+        @copy
       end
 
-      conf.elements.select {|e|
-        e.name == 'route'
-      }.each {|e|
-        route = Route.new(e.arg, router)
-        route.configure(e)
-        @routes << route
-      }
+      def emit(tag, es)
+        tag = @tag_modifier.call(tag)
+        @router.emit_stream(tag, es)
+      end
     end
 
-    def emit(tag, es, chain)
-      ntag, targets = @match_cache[tag]
+    def process(tag, es)
+      modified_tag, targets = @match_cache[tag]
       unless targets
-        ntag = tag.sub(@prefix_match, @tag_prefix)
+        modified_tag = @default_tag_modifier ? @default_tag_modifier.call(tag) : tag
         targets = []
-        @routes.each {|r|
-          if r.match?(ntag)
+        @routes.each do |r|
+          if r.match?(modified_tag)
             targets << r
             break unless r.copy?
           end
-        }
-        if @match_cache.size < 1024  # TODO size limit
-          @match_cache[tag] = [ntag, targets]
+        end
+
+        @mutex.synchronize do
+          if @match_cache.size >= @match_cache_size
+            remove_keys = @match_cache.keys[0...(@match_cache_size / 2)]
+            @match_cache.delete_if{|key, _value| remove_keys.include?(key) }
+          end
+          @match_cache[tag] = [modified_tag, targets]
         end
       end
 
       case targets.size
       when 0
-        return
+        # do nothing
       when 1
-        targets.first.emit(ntag, es)
-        chain.next
+        targets.first.emit(modified_tag, es)
       else
-        unless es.repeatable?
-          m = MultiEventStream.new
-          es.each {|time,record|
-            m.add(time, record)
-          }
-          es = m
+        targets.each do |target|
+          dup_es = if es.respond_to?(:dup)
+                     es.dup
+                   else
+                     m_es = MultiEventStream.new
+                     es.each{|t,r| m_es.add(t, r) }
+                     m_es
+                   end
+          target.emit(modified_tag, dup_es)
         end
-        targets.each {|t|
-          t.emit(ntag, es)
-        }
-        chain.next
       end
     end
   end

data/test/plugin/test_out_route.rb

@@ -1,6 +1,12 @@
-require 'fluent/test'
 require 'fluent/plugin/out_route'
 
+require 'fluent/test'
+require 'fluent/test/driver/multi_output'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/test/helpers'
+include Fluent::Test::Helpers
+
 class RouteOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
@@ -25,14 +31,15 @@ class RouteOutputTest < Test::Unit::TestCase
     </route>
   ]
 
-  def create_driver(conf, tag)
-    d = Fluent::Test::OutputTestDriver.new(Fluent::RouteOutput, tag)
+  def create_driver(conf)
+    d = Fluent::Test::Driver::BaseOwner.new(Fluent::Plugin::RouteOutput)
+    d.extend(Fluent::Test::Driver::EventFeeder)
     Fluent::Engine.root_agent.define_singleton_method(:find_label) do |label_name|
       obj = Object.new
       obj.define_singleton_method(:event_router){ d.instance.router } # for test...
       obj
     end
-    d.configure(conf, true)
+    d.configure(conf)
   end
 
   def test_configure
@@ -40,52 +47,52 @@ class RouteOutputTest < Test::Unit::TestCase
   end
 
   def test_emit_t1
-    d = create_driver(CONFIG, "t.t1.test")
+    d = create_driver(CONFIG)
 
-    time = Time.parse("2011-11-11 11:11:11 UTC").to_i
-    d.run do
-      d.emit({"a" => 1}, time)
-      d.emit({"a" => 2}, time)
+    time = event_time("2011-11-11 11:11:11 UTC")
+    d.run(default_tag: "t.t1.test", expect_records: 2) do
+      d.feed(time, {"a" => 1})
+      d.feed(time, {"a" => 2})
     end
 
-    emits = d.emits
-    assert_equal 2, emits.size
+    events = d.events
+    assert_equal 2, events.size
 
-    assert_equal ["yay.test", time, {"a" => 1}], emits[0]
-    assert_equal ["yay.test", time, {"a" => 2}], emits[1]
+    assert_equal ["yay.test", time, {"a" => 1}], events[0]
+    assert_equal ["yay.test", time, {"a" => 2}], events[1]
   end
 
   def test_emit_t2
-    d = create_driver(CONFIG, "t.t2.test")
+    d = create_driver(CONFIG)
 
-    time = Time.parse("2011-11-11 11:11:11 UTC").to_i
-    d.run do
-      d.emit({"a" => 1}, time)
-      d.emit({"a" => 2}, time)
+    time = event_time("2011-11-11 11:11:11 UTC")
+    d.run(default_tag: "t.t2.test", expect_records: 2) do
+      d.feed(time, {"a" => 1})
+      d.feed(time, {"a" => 2})
     end
 
-    emits = d.emits
-    assert_equal 2, emits.size
+    events = d.events
+    assert_equal 2, events.size
 
-    assert_equal ["foo.test", time, {"a" => 1}], emits[0]
-    assert_equal ["foo.test", time, {"a" => 2}], emits[1]
+    assert_equal ["foo.test", time, {"a" => 1}], events[0]
+    assert_equal ["foo.test", time, {"a" => 2}], events[1]
   end
 
   def test_emit_others
-    d = create_driver(CONFIG, "t.t3.test")
+    d = create_driver(CONFIG)
 
-    time = Time.parse("2011-11-11 11:11:11 UTC").to_i
-    d.run do
-      d.emit({"a" => 1}, time)
-      d.emit({"a" => 2}, time)
+    time = event_time("2011-11-11 11:11:11 UTC")
+    d.run(default_tag: "t.t3.test", expect_records: 4) do
+      d.feed(time, {"a" => 1})
+      d.feed(time, {"a" => 2})
     end
 
-    emits = d.emits
-    assert_equal 4, emits.size
+    events = d.events
+    assert_equal 4, events.size
 
-    assert_equal ["t3.test", time, {"a" => 1}], emits[0]
-    assert_equal ["t3.test", time, {"a" => 1}], emits[1]
-    assert_equal ["t3.test", time, {"a" => 2}], emits[2]
-    assert_equal ["t3.test", time, {"a" => 2}], emits[3]
+    assert_equal ["t3.test", time, {"a" => 1}], events[0]
+    assert_equal ["t3.test", time, {"a" => 1}], events[1]
+    assert_equal ["t3.test", time, {"a" => 2}], events[2]
+    assert_equal ["t3.test", time, {"a" => 2}], events[3]
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-route
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -9,20 +9,20 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-31 00:00:00.000000000 Z
+date: 2017-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.0
 - !ruby/object:Gem::Dependency