fluentd 0.14.0 → 0.14.1

data/lib/fluent/process.rb

@@ -14,10 +14,15 @@
 #    limitations under the License.
 #
 
+# This feature will be deprecated after introducing
+#  symmetric multi processing in core.
+
 require 'thread'
 
 require 'fluent/config'
-require 'fluent/engine'
+## This comment out (to remove circular reference) is a bit risky,
+## but in all known cases this file is required after 'fluent/engine'.
+# require 'fluent/engine'
 require 'fluent/event'
 
 module Fluent

data/lib/fluent/registry.rb

@@ -18,7 +18,7 @@ require 'fluent/config/error'
 
 module Fluent
   class Registry
-    DEFAULT_PLUGIN_PATH = File.expand_path('plugin', __FILE__)
+    DEFAULT_PLUGIN_PATH = File.expand_path('../plugin', __FILE__)
 
     def initialize(kind, search_prefix)
       @kind = kind

data/lib/fluent/supervisor.rb

@@ -201,7 +201,9 @@ module Fluent
       logger_initializer.init
       logger = $log
 
-      daemonize = params.fetch('daemonize', false)
+      # ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
+      pid_path = params['daemonize']
+      daemonize = !!params['daemonize']
       main_cmd = params['main_cmd']
 
       se_config = {
@@ -232,6 +234,9 @@ module Fluent
           fluentd_conf: fluentd_conf,
           main_cmd: main_cmd,
       }
+      if daemonize
+        se_config[:pid_path] = pid_path
+      end
       pre_params = params.dup
       params['pre_loadtime'] = Time.now.to_i
       params['pre_config_mtime'] = config_mtime
@@ -306,6 +311,7 @@ module Fluent
         without_source: false,
         use_v1_config: true,
         supervise: true,
+        standalone_worker: false,
         signame: nil,
         winsvcreg: nil,
       }
@@ -314,6 +320,7 @@ module Fluent
     def initialize(opt)
       @daemonize = opt[:daemonize]
       @supervise = opt[:supervise]
+      @standalone_worker= opt[:standalone_worker]
       @config_path = opt[:config_path]
       @inline_config = opt[:inline_config]
       @use_v1_config = opt[:use_v1_config]
@@ -376,6 +383,7 @@ module Fluent
       $log.info "starting fluentd-#{Fluent::VERSION} without supervision"
 
       main_process do
+        create_socket_manager if @standalone_worker
         change_privilege
         init_engine
         run_configure
@@ -386,6 +394,12 @@ module Fluent
 
     private
 
+    def create_socket_manager
+      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      ServerEngine::SocketManager::Server.open(socket_manager_path)
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+    end
+
     def dry_run
       $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode"
       change_privilege
@@ -425,21 +439,25 @@ module Fluent
     def supervise
       $log.info "starting fluentd-#{Fluent::VERSION}"
 
+      rubyopt = ENV["RUBYOPT"]
       if Fluent.windows?
+        # Shellwords doesn't work on windows, then used gsub for adapting space char instead of Shellwords
         fluentd_spawn_cmd = ServerEngine.ruby_bin_path + " -Eascii-8bit:ascii-8bit "
+        fluentd_spawn_cmd << ' "' + rubyopt.gsub('"', '""') + '" ' if rubyopt
         fluentd_spawn_cmd << ' "' + $0.gsub('"', '""') + '" '
         $fluentdargv.each{|a|
           fluentd_spawn_cmd << ('"' + a.gsub('"', '""') + '" ')
         }
       else
         fluentd_spawn_cmd = ServerEngine.ruby_bin_path + " -Eascii-8bit:ascii-8bit "
+        fluentd_spawn_cmd << ' ' + rubyopt + ' ' if rubyopt
         fluentd_spawn_cmd << $0.shellescape + ' '
         $fluentdargv.each{|a|
           fluentd_spawn_cmd << (a.shellescape + " ")
         }
       end
 
-      fluentd_spawn_cmd << ("--no-supervisor")
+      fluentd_spawn_cmd << ("--under-supervisor")
       $log.info "spawn command to main: " + fluentd_spawn_cmd
 
       params = {}

data/lib/fluent/test.rb

@@ -16,6 +16,7 @@
 
 require 'test/unit'
 require 'fluent/env' # for Fluent.windows?
+require 'fluent/test/log'
 require 'fluent/test/base'
 require 'fluent/test/input_test'
 require 'fluent/test/output_test'
@@ -24,8 +25,32 @@ require 'fluent/test/parser_test'
 require 'fluent/test/formatter_test'
 require 'serverengine'
 
-dl_opts = {}
-dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO
-logdev = Fluent::Test::DummyLogDevice.new
-logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
-$log ||= Fluent::Log.new(logger)
+
+module Fluent
+  module Test
+    def self.dummy_logger
+      dl_opts = {log_level: ServerEngine::DaemonLogger::INFO}
+      logdev = Fluent::Test::DummyLogDevice.new
+      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+      Fluent::Log.new(logger)
+    end
+
+    def self.setup
+      $log = dummy_logger
+
+      Fluent.__send__(:remove_const, :Engine)
+      engine = Fluent.const_set(:Engine, EngineClass.new).init(SystemConfig.new)
+
+      engine.define_singleton_method(:now=) {|n|
+        @now = n
+      }
+      engine.define_singleton_method(:now) {
+        @now ||= super()
+      }
+
+      nil
+    end
+  end
+end
+
+$log ||= Fluent::Test.dummy_logger

data/lib/fluent/test/base.rb

@@ -14,27 +14,14 @@
 #    limitations under the License.
 #
 
+require 'fluent/config'
 require 'fluent/engine'
 require 'fluent/system_config'
-require 'fluent/config'
+require 'fluent/test/log'
 require 'serverengine'
 
 module Fluent
   module Test
-    def self.setup
-      Fluent.__send__(:remove_const, :Engine)
-      engine = Fluent.const_set(:Engine, EngineClass.new).init(SystemConfig.new)
-
-      engine.define_singleton_method(:now=) {|n|
-        @now = n
-      }
-      engine.define_singleton_method(:now) {
-        @now ||= super()
-      }
-
-      nil
-    end
-
     class TestDriver
       include ::Test::Unit::Assertions
 
@@ -84,57 +71,6 @@ module Fluent
         end
       end
     end
-
-    class DummyLogDevice
-      attr_reader :logs
-
-      def initialize
-        @logs = []
-      end
-
-      def reset
-        @logs = []
-      end
-
-      def tty?
-        false
-      end
-
-      def puts(*args)
-        args.each{ |arg| write(arg + "\n") }
-      end
-
-      def write(message)
-        @logs.push message
-      end
-
-      def flush
-        true
-      end
-
-      def close
-        true
-      end
-    end
-
-    class TestLogger < Fluent::PluginLogger
-      def initialize
-        @logdev = DummyLogDevice.new
-        dl_opts = {}
-        dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO
-        logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
-        log = Fluent::Log.new(logger)
-        super(log)
-      end
-
-      def reset
-        @logdev.reset
-      end
-
-      def logs
-        @logdev.logs
-      end
-    end
   end
 end
 

data/lib/fluent/test/driver/base.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/config/element'
+require 'fluent/log'
 require 'fluent/test/driver/test_event_router'
 
 require 'timeout'
@@ -104,6 +106,7 @@ module Fluent
         end
 
         def events(tag: nil)
+          return [] if @event_streams.nil?
           selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }
           if block_given?
             selected.each do |e|

data/lib/fluent/test/driver/base_owned.rb

@@ -0,0 +1,106 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/config/element'
+require 'fluent/log'
+require 'fluent/test/driver/owner'
+
+module Fluent
+  module Test
+    module Driver
+      class BaseOwned
+        def initialize(klass, opts: {}, &block)
+          if klass.is_a?(Class)
+            if block
+              # Create new class for test w/ overwritten methods
+              #   klass.dup is worse because its ancestors does NOT include original class name
+              klass = Class.new(klass)
+              klass.module_eval(&block)
+            end
+            @instance = klass.new
+          else
+            @instance = klass
+          end
+          owner = Fluent::Test::Driver::Owner.new
+          if opts
+            owner.system_config_override(opts)
+          end
+          owner.log = TestLogger.new
+
+          if @instance.respond_to?(:owner=)
+            @instance.owner = owner
+            if opts
+              @instance.system_config_override(opts)
+            end
+          end
+
+          @logs = owner.log.out.logs
+          @section_name = ''
+        end
+
+        attr_reader :instance, :logs
+
+        def configure(conf, syntax: :v1)
+          if conf.is_a?(Fluent::Config::Element)
+            @config = conf
+          elsif conf.is_a?(Hash)
+            @config = Fluent::Config::Element.new(@section_name, "", Hash[conf.map{|k,v| [k.to_s, v]}], [])
+          else
+            @config = Fluent::Config.parse(conf, @section_name, "", syntax: syntax)
+          end
+          @instance.configure(@config)
+          self
+        end
+
+        def run(start: true, shutdown: true, &block)
+          instance_start if start
+
+          begin
+            yield
+          ensure
+            instance_shutdown if shutdown
+          end
+        end
+
+        def instance_start
+          unless @instance.started?
+            @instance.start
+            instance_hook_after_started
+          end
+        end
+
+        def instance_hook_after_started
+          # insert hooks for tests available after instance.start
+        end
+
+        def instance_shutdown
+          @instance.stop            unless @instance.stopped?
+          @instance.before_shutdown unless @instance.before_shutdown?
+          @instance.shutdown        unless @instance.shutdown?
+
+          if @instance.respond_to?(:event_loop_wait_until_stop)
+            @instance.event_loop_wait_until_stop
+          end
+
+          @instance.after_shutdown  unless @instance.after_shutdown?
+          @instance.close     unless @instance.closed?
+          @instance.terminate unless @instance.terminated?
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/formatter.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owned'
+
+module Fluent
+  module Test
+    module Driver
+      class Formatter < BaseOwned
+        def initialize(klass, **kwargs, &block)
+          super
+          @section_name = "format"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/multi_output.rb

@@ -0,0 +1,52 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/plugin/multi_output'
+
+module Fluent
+  module Test
+    module Driver
+      class MultiOutput < Base
+        include EventFeeder
+
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::MultiOutput" unless @instance.is_a? Fluent::Plugin::MultiOutput
+          @flush_buffer_at_cleanup = nil
+        end
+
+        def run(flush: true, **kwargs, &block)
+          @flush_buffer_at_cleanup = flush
+          super(**kwargs, &block)
+        end
+
+        def run_actual(**kwargs, &block)
+          super(**kwargs, &block)
+          if @flush_buffer_at_cleanup
+            @instance.outputs.each{|o| o.force_flush }
+          end
+        end
+
+        def flush
+          @instance.outputs.each{|o| o.force_flush }
+        end
+      end
+    end
+  end
+end