fluentd 0.14.1 → 0.14.2

data/lib/fluent/registry.rb

@@ -20,9 +20,10 @@ module Fluent
   class Registry
     DEFAULT_PLUGIN_PATH = File.expand_path('../plugin', __FILE__)
 
-    def initialize(kind, search_prefix)
+    def initialize(kind, search_prefix, dir_search_prefix: nil)
       @kind = kind
       @search_prefix = search_prefix
+      @dir_search_prefix = dir_search_prefix
       @map = {}
       @paths = [DEFAULT_PLUGIN_PATH]
     end
@@ -54,11 +55,10 @@ module Fluent
     end
 
     def search(type)
-      path = "#{@search_prefix}#{type}"
-
-      # prefer LOAD_PATH than gems
-      [@paths, $LOAD_PATH].each do |paths|
-        files = paths.map { |lp|
+      # search from additional plugin directories
+      if @dir_search_prefix
+        path = "#{@dir_search_prefix}#{type}"
+        files = @paths.map { |lp|
           lpath = File.expand_path(File.join(lp, "#{path}.rb"))
           File.exist?(lpath) ? lpath : nil
         }.compact
@@ -69,6 +69,19 @@ module Fluent
         end
       end
 
+      path = "#{@search_prefix}#{type}"
+
+      # prefer LOAD_PATH than gems
+      files = $LOAD_PATH.map { |lp|
+        lpath = File.expand_path(File.join(lp, "#{path}.rb"))
+        File.exist?(lpath) ? lpath : nil
+      }.compact
+      unless files.empty?
+        # prefer newer version
+        require files.sort.last
+        return
+      end
+
       specs = Gem::Specification.find_all { |spec|
         spec.contains_requirable_file? path
       }

data/lib/fluent/supervisor.rb

@@ -75,6 +75,16 @@ module Fluent
         Process.kill :TERM, $$
         nil
       }
+      @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
+        $log.debug "fluentd RPC got /api/processes.flushBuffersAndKillWorkers request"
+        if Fluent.windows?
+          $log.warn "operation 'flushBuffersAndKillWorkers' is not supported on Windows now."
+        else
+          Process.kill :USR1, $$
+          Process.kill :TERM, $$
+        end
+        nil
+      }
       @rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
         $log.debug "fluentd RPC got /api/plugins.flushBuffers request"
         unless Fluent.windows?
@@ -114,12 +124,21 @@ module Fluent
     end
 
     def install_supervisor_signal_handlers
+      trap :HUP do
+        $log.debug "fluentd supervisor process get SIGHUP"
+        supervisor_sighup_handler
+      end unless Fluent.windows?
+
       trap :USR1 do
         $log.debug "fluentd supervisor process get SIGUSR1"
         supervisor_sigusr1_handler
       end unless Fluent.windows?
     end
 
+    def supervisor_sighup_handler
+      kill_worker
+    end
+
     def supervisor_sigusr1_handler
       if log = config[:logger_initializer]
         log.reopen!
@@ -136,7 +155,7 @@ module Fluent
         if Fluent.windows?
           Process.kill :KILL, pid
         else
-          Process.kill :INT, pid
+          Process.kill :TERM, pid
         end
       end
     end
@@ -487,6 +506,13 @@ module Fluent
       # worker process SHOULD NOT do anything with SIGINT, SHOULD just ignore.
       trap :INT do
         $log.debug "fluentd main process get SIGINT"
+
+        # When Fluentd is launched without supervisor, worker should handle ctrl-c by itself
+        if @standalone_worker
+          @finished = true
+          $log.debug "getting start to shutdown main process"
+          Fluent::Engine.stop
+        end
       end
 
       trap :TERM do

data/lib/fluent/test/driver/base.rb

@@ -14,9 +14,9 @@
 #    limitations under the License.
 #
 
+require 'fluent/config'
 require 'fluent/config/element'
 require 'fluent/log'
-require 'fluent/test/driver/test_event_router'
 
 require 'timeout'
 
@@ -24,6 +24,8 @@ module Fluent
   module Test
     module Driver
       class Base
+        attr_reader :instance, :logs
+
         def initialize(klass, opts: {}, &block)
           if klass.is_a?(Class)
             if block
@@ -36,47 +38,16 @@ module Fluent
           else
             @instance = klass
           end
-          if opts
-            @instance.system_config_override(opts)
-          end
-          @instance.log = TestLogger.new
-          @logs = @instance.log.out.logs
+
+          @logs = []
 
           @run_post_conditions = []
           @run_breaking_conditions = []
-
           @broken = false
-
-          @event_streams = nil
-          @error_events = nil
         end
 
-        attr_reader :instance, :logs
-
         def configure(conf, syntax: :v1)
-          if conf.is_a?(Fluent::Config::Element)
-            @config = conf
-          else
-            @config = Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
-          end
-
-          if @instance.respond_to?(:router=)
-            @event_streams = []
-            @error_events = []
-
-            driver = self
-            mojule = Module.new do
-              define_method(:event_emitter_router) do |label_name|
-                TestEventRouter.new(driver)
-              end
-            end
-            @instance.singleton_class.module_eval do
-              prepend mojule
-            end
-          end
-
-          @instance.configure(@config)
-          self
+          raise NotImplementedError
         end
 
         def end_if(&block)
@@ -93,50 +64,7 @@ module Fluent
           @broken
         end
 
-        Emit = Struct.new(:tag, :es)
-        ErrorEvent = Struct.new(:tag, :time, :record, :error)
-
-        # via TestEventRouter
-        def emit_event_stream(tag, es)
-          @event_streams << Emit.new(tag, es)
-        end
-
-        def emit_error_event(tag, time, record, error)
-          @error_events << ErrorEvent.new(tag, time, record, error)
-        end
-
-        def events(tag: nil)
-          return [] if @event_streams.nil?
-          selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }
-          if block_given?
-            selected.each do |e|
-              e.es.each do |time, record|
-                yield e.tag, time, record
-              end
-            end
-          else
-            list = []
-            selected.each do |e|
-              e.es.each do |time, record|
-                list << [e.tag, time, record]
-              end
-            end
-            list
-          end
-        end
-
-        def error_events(tag: nil)
-          selected = @error_events.select{|e| tag.nil? ? true : e.tag == tag }
-          if block_given?
-            selected.each do |e|
-              yield e.tag, e.time, e.record, e.error
-            end
-          else
-            selected.map{|e| [e.tag, e.time, e.record, e.error] }
-          end
-        end
-
-        def run(expect_emits: nil, expect_records: nil, timeout: nil, start: true, shutdown: true, &block)
+        def run(timeout: nil, start: true, shutdown: true, &block)
           instance_start if start
 
           if @instance.respond_to?(:thread_wait_until_start)
@@ -147,7 +75,7 @@ module Fluent
           end
 
           begin
-            run_actual(expect_emits: expect_emits, expect_records: expect_records, timeout: timeout, &block)
+            run_actual(timeout: timeout, &block)
           ensure
             instance_shutdown if shutdown
           end
@@ -183,7 +111,7 @@ module Fluent
           @instance.terminate unless @instance.terminated?
         end
 
-        def run_actual(expect_emits: nil, expect_records: nil, timeout: nil, &block)
+        def run_actual(timeout: nil, &block)
           if @instance.respond_to?(:_threads)
             until @instance._threads.values.all?(&:alive?)
               sleep 0.01
@@ -196,12 +124,6 @@ module Fluent
             end
           end
 
-          if expect_emits
-            @run_post_conditions << ->(){ @emit_streams.size >= expect_emits }
-          end
-          if expect_records
-            @run_post_conditions << ->(){ @emit_streams.reduce(0){|a, e| a + e.es.size } >= expected_records }
-          end
           if timeout
             stop_at = Time.now + timeout
             @run_breaking_conditions << ->(){ Time.now >= stop_at }
@@ -211,20 +133,22 @@ module Fluent
             raise ArgumentError, "no stop conditions nor block specified"
           end
 
-          if !block_given?
-            block = ->(){ sleep(0.1) until stop? }
-          end
+          proc = if block_given?
+                   ->(){ block.call; sleep(0.1) until stop? }
+                 else
+                   ->(){ sleep(0.1) until stop? }
+                 end
 
           if timeout
             begin
               Timeout.timeout(timeout * 1.1) do |sec|
-                block.call
+                proc.call
               end
             rescue Timeout::Error
               @broken = true
             end
           else
-            block.call
+            proc.call
           end
         end
 

data/lib/fluent/test/driver/base_owned.rb

@@ -14,28 +14,29 @@
 #    limitations under the License.
 #
 
-require 'fluent/config'
-require 'fluent/config/element'
+require 'fluent/test/driver/base'
+
+require 'fluent/plugin/base'
+require 'fluent/plugin_id'
 require 'fluent/log'
-require 'fluent/test/driver/owner'
+require 'fluent/plugin_helper'
 
 module Fluent
   module Test
     module Driver
-      class BaseOwned
+      class OwnerDummy < Fluent::Plugin::Base
+        include PluginId
+        include PluginLoggerMixin
+        include PluginHelper::Mixin
+      end
+
+      class BaseOwned < Base
+        attr_accessor :section_name
+
         def initialize(klass, opts: {}, &block)
-          if klass.is_a?(Class)
-            if block
-              # Create new class for test w/ overwritten methods
-              #   klass.dup is worse because its ancestors does NOT include original class name
-              klass = Class.new(klass)
-              klass.module_eval(&block)
-            end
-            @instance = klass.new
-          else
-            @instance = klass
-          end
-          owner = Fluent::Test::Driver::Owner.new
+          super
+
+          owner = OwnerDummy.new
           if opts
             owner.system_config_override(opts)
           end
@@ -52,8 +53,6 @@ module Fluent
           @section_name = ''
         end
 
-        attr_reader :instance, :logs
-
         def configure(conf, syntax: :v1)
           if conf.is_a?(Fluent::Config::Element)
             @config = conf
@@ -65,41 +64,6 @@ module Fluent
           @instance.configure(@config)
           self
         end
-
-        def run(start: true, shutdown: true, &block)
-          instance_start if start
-
-          begin
-            yield
-          ensure
-            instance_shutdown if shutdown
-          end
-        end
-
-        def instance_start
-          unless @instance.started?
-            @instance.start
-            instance_hook_after_started
-          end
-        end
-
-        def instance_hook_after_started
-          # insert hooks for tests available after instance.start
-        end
-
-        def instance_shutdown
-          @instance.stop            unless @instance.stopped?
-          @instance.before_shutdown unless @instance.before_shutdown?
-          @instance.shutdown        unless @instance.shutdown?
-
-          if @instance.respond_to?(:event_loop_wait_until_stop)
-            @instance.event_loop_wait_until_stop
-          end
-
-          @instance.after_shutdown  unless @instance.after_shutdown?
-          @instance.close     unless @instance.closed?
-          @instance.terminate unless @instance.terminated?
-        end
       end
     end
   end

data/lib/fluent/test/driver/base_owner.rb

@@ -0,0 +1,125 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base'
+require 'fluent/test/driver/test_event_router'
+
+module Fluent
+  module Test
+    module Driver
+      class BaseOwner < Base
+        def initialize(klass, opts: {}, &block)
+          super
+
+          if opts
+            @instance.system_config_override(opts)
+          end
+          @instance.log = TestLogger.new
+          @logs = @instance.log.out.logs
+
+          @event_streams = nil
+          @error_events = nil
+        end
+
+        def configure(conf, syntax: :v1)
+          if conf.is_a?(Fluent::Config::Element)
+            @config = conf
+          else
+            @config = Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
+          end
+
+          if @instance.respond_to?(:router=)
+            @event_streams = []
+            @error_events = []
+
+            driver = self
+            mojule = Module.new do
+              define_method(:event_emitter_router) do |label_name|
+                TestEventRouter.new(driver)
+              end
+            end
+            @instance.singleton_class.prepend mojule
+          end
+
+          @instance.configure(@config)
+          self
+        end
+
+        Emit = Struct.new(:tag, :es)
+        ErrorEvent = Struct.new(:tag, :time, :record, :error)
+
+        # via TestEventRouter
+        def emit_event_stream(tag, es)
+          @event_streams << Emit.new(tag, es)
+        end
+
+        def emit_error_event(tag, time, record, error)
+          @error_events << ErrorEvent.new(tag, time, record, error)
+        end
+
+        def events(tag: nil)
+          return [] if @event_streams.nil?
+          selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }
+          if block_given?
+            selected.each do |e|
+              e.es.each do |time, record|
+                yield e.tag, time, record
+              end
+            end
+          else
+            list = []
+            selected.each do |e|
+              e.es.each do |time, record|
+                list << [e.tag, time, record]
+              end
+            end
+            list
+          end
+        end
+
+        def emit_count
+          @event_streams.size
+        end
+
+        def record_count
+          @event_streams.reduce(0) {|a, e| a + e.es.size }
+        end
+
+        def error_events(tag: nil)
+          selected = @error_events.select{|e| tag.nil? ? true : e.tag == tag }
+          if block_given?
+            selected.each do |e|
+              yield e.tag, e.time, e.record, e.error
+            end
+          else
+            selected.map{|e| [e.tag, e.time, e.record, e.error] }
+          end
+        end
+
+        def run(expect_emits: nil, expect_records: nil, timeout: nil, start: true, shutdown: true, &block)
+          if expect_emits
+            @run_post_conditions << ->(){ emit_count >= expect_emits }
+          end
+          if expect_records
+            @run_post_conditions << ->(){ record_count >= expect_records }
+          end
+
+          super(timeout: timeout, start: start, shutdown: shutdown, &block)
+        end
+      end
+    end
+  end
+end