fluentd 0.14.14 → 0.14.15

data/lib/fluent/plugin/out_file.rb

@@ -105,6 +105,11 @@ module Fluent::Plugin
         buffer_conf['path'] = conf['path'] || '/tmp/dummy_path'
       end
 
+      if conf.has_key?('utc') || conf.has_key?('localtime')
+        param_name = conf.has_key?('utc') ? 'utc' : 'localtime'
+        log.warn "'#{param_name}' is deperecated for output plugin. This parameter is used for formatter plugin in compatibility layer. If you want to use same feature, use timekey_use_utc parameter in <buffer> directive instead"
+      end
+
       super
 
       @compress_method = SUPPORTED_COMPRESS_MAP[@compress]

data/lib/fluent/plugin/output.rb

@@ -122,12 +122,17 @@ module Fluent
         raise NotImplementedError, "BUG: output plugins MUST implement this method"
       end
 
-      def formatted_to_msgpack_binary
+      def formatted_to_msgpack_binary?
         # To indicate custom format method (#format) returns msgpack binary or not.
         # If #format returns msgpack binary, override this method to return true.
         false
       end
 
+      # Compatibility for existing plugins
+      def formatted_to_msgpack_binary
+        formatted_to_msgpack_binary?
+      end
+
       def prefer_buffered_processing
         # override this method to return false only when all of these are true:
         #  * plugin has both implementation for buffered and non-buffered methods

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -55,7 +55,7 @@ module Fluent
         "keys" => "keys", # CSVParser, TSVParser (old ValuesParser)
         "time_key"    => "time_key",
         "time_format" => "time_format",
-        "localtim" => nil,
+        "localtime" => nil,
         "utc" => nil,
         "delimiter"   => "delimiter",
         "keep_time_key" => "keep_time_key",

data/lib/fluent/root_agent.rb

@@ -64,11 +64,34 @@ module Fluent
     attr_reader :labels
 
     def configure(conf)
+      # initialize <worker> elements
+      conf.elements(name: 'worker').each do |e|
+        target_worker_id_str = e.arg
+        if target_worker_id_str.empty?
+          raise ConfigError, "Missing worker id on <worker> directive"
+        end
+
+        target_worker_id = target_worker_id_str.to_i
+        if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
+          raise ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
+        end
+
+        e.elements.each do |elem|
+          unless ['source', 'match', 'filter', 'label'].include?(elem.name)
+            raise ConfigError, "<worker> section cannot have <#{elem.name}> directive"
+          end
+          elem.set_target_worker_id(target_worker_id)
+        end
+        conf += e
+      end
+      conf.elements.delete_if{|e| e.name == 'worker'}
+
       error_label_config = nil
 
       # initialize <label> elements before configuring all plugins to avoid 'label not found' in input, filter and output.
       label_configs = {}
       conf.elements(name: 'label').each { |e|
+        next if e.for_another_worker?
         name = e.arg
         raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
 
@@ -90,6 +113,7 @@ module Fluent
         log.info :worker0, "'--without-source' is applied. Ignore <source> sections"
       else
         conf.elements(name: 'source').each { |e|
+          next if e.for_another_worker?
           type = e['@type']
           raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
           add_source(type, e)
@@ -235,7 +259,8 @@ module Fluent
     end
 
     def add_source(type, conf)
-      log.info :worker0, "adding source", type: type
+      log_type = conf.for_this_worker? ? :default : :worker0
+      log.info log_type, "adding source", type: type
 
       input = Plugin.new_input(type)
       # <source> emits events to the top-level event router (RootAgent#event_router).

data/lib/fluent/supervisor.rb

@@ -384,7 +384,6 @@ module Fluent
         supervise: true,
         standalone_worker: false,
         signame: nil,
-        winsvcreg: nil,
       }
     end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.14'
+  VERSION = '0.14.15'
 
 end

data/lib/fluent/winsvc.rb

@@ -14,10 +14,9 @@
 #    limitations under the License.
 #
 
-INTEVENTOBJ_NAME = "fluentdwinsvc"
-
 begin
 
+  require 'optparse'
   require 'windows/debug'
   require 'Windows/Library'
   require 'win32/daemon'
@@ -27,34 +26,50 @@ begin
   include Windows::Library
   include Windows::Debug
 
-  def read_fluentdopt
+  op = OptionParser.new
+  opts = {service_name: nil}
+  op.on('--service-name NAME', "The name of the Windows Service") {|name|
+    opts[:service_name] = name
+  }
+  op.parse(ARGV)
+  if opts[:service_name] == nil
+    raise "Error: No Windows Service name set. Use '--service-name'"
+  end 
+
+  def read_fluentdopt(service_name)
     require 'win32/Registry'
-    Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\fluentdwinsvc") do |reg|
+    Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
       reg.read("fluentdopt")[1] rescue ""
     end
   end
 
-  def service_main_start
+  def service_main_start(service_name)
     ruby_path = 0.chr * 260
     GetModuleFileName.call(0, ruby_path,260)
     ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
     rubybin_dir = ruby_path[0, ruby_path.rindex("/")]
-    opt = read_fluentdopt
-    Process.spawn(rubybin_dir + "/ruby.exe " + rubybin_dir + "/fluentd " + opt + " -x " + INTEVENTOBJ_NAME)
+    opt = read_fluentdopt(service_name)
+    Process.spawn("\"#{rubybin_dir}/ruby.exe\" \"#{rubybin_dir}/fluentd\" #{opt} -x #{service_name}")
   end
 
   class FluentdService < Daemon
     @pid = 0
+    @service_name = ''
 
+    def initialize(service_name)
+      @service_name = service_name
+    end
+    
     def service_main
-      @pid = service_main_start
+    
+      @pid = service_main_start(@service_name)
       while running?
         sleep 10
       end
     end
 
     def service_stop
-      ev = Win32::Event.open(INTEVENTOBJ_NAME)
+      ev = Win32::Event.open(@service_name)
       ev.set
       ev.close
       if @pid > 0
@@ -63,9 +78,8 @@ begin
     end
   end
 
-  FluentdService.mainloop
+  FluentdService.new(opts[:service_name]).mainloop
 
 rescue Exception => err
   raise
 end
-

data/test/command/test_fluentd.rb

@@ -297,7 +297,7 @@ CONF
       assert_log_matches(
         create_cmdline(conf_path),
         "fluentd worker is now running",
-        '[warn]: match for some tags of log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
+        '[warn]: #0 match for some tags of log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
         '[warn]: #0 no patterns matched tag="fluent.info"',
       )
     end
@@ -334,7 +334,7 @@ CONF
       assert_log_matches(
         create_cmdline(conf_path),
         "fluentd worker is now running",
-        '[warn]: match for some tags of log events are not defined (to be ignored) tags=["fluent.info", "fluent.fatal"]',
+        '[warn]: #0 match for some tags of log events are not defined (to be ignored) tags=["fluent.info", "fluent.fatal"]',
         patterns_not_match: ['[warn]: no patterns matched tag="fluent.info"'],
       )
     end
@@ -614,5 +614,211 @@ CONF
         "config error file=\"#{conf_path}\" error_class=Fluent::ConfigError error=\"Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)\"",
       )
     end
+
+    test 'failed to start workers when configured plugins as chidren of MultiOutput do not support multi worker configuration' do
+      script = <<-EOC
+require 'fluent/plugin/output'
+module Fluent::Plugin
+  class SingleOutput < Output
+    Fluent::Plugin.register_output('single', self)
+    def multi_workers_ready?
+      false
+    end
+    def write(chunk)
+    end
+  end
+end
+EOC
+      plugin_path = create_plugin_file('out_single.rb', script)
+
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<source>
+  @type single
+  @id single
+  @label @dummydata
+</source>
+<label @dummydata>
+  <match dummy>
+    @type copy
+    <store>
+      @type single
+    </store>
+    <store>
+      @type single
+    </store>
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('workers_invalid3.conf', conf)
+      assert_fluentd_fails_to_start(
+        create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
+        "Plugin 'single' does not support multi workers configuration (Fluent::Plugin::SingleOutput)",
+      )
+    end
+
+    test 'success to start a worker with worker specific configuration' do
+      conf = <<CONF
+<system>
+  workers 2
+  root_dir #{@root_path}
+</system>
+<source>
+  @type dummy
+  @id dummy
+  @label @dummydata
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<worker 1>
+  <source>
+    @type dummy
+    @id dummy_in_worker
+    @label @dummydata
+    tag dummy
+    dummy {"message": "yay!"}
+  </source>
+</worker>
+<label @dummydata>
+  <match dummy>
+    @type null
+    @id   blackhole
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('worker_section0.conf', conf)
+      assert Dir.exist?(@root_path)
+
+      assert_log_matches(
+        create_cmdline(conf_path),
+        "#0 fluentd worker is now running worker=0",
+        "#1 fluentd worker is now running worker=1",
+        /(?!#\d) adding source type="dummy"/,
+        '#1 adding source type="dummy"'
+      )
+    end
+
+    test 'success to start workers when configured plugins only for specific worker do not support multi worker configuration' do
+      script =  <<-EOC
+require 'fluent/plugin/input'
+module Fluent::Plugin
+  class SingleInput < Input
+    Fluent::Plugin.register_input('single', self)
+    def multi_workers_ready?
+      false
+    end
+  end
+end
+EOC
+      plugin_path = create_plugin_file('in_single.rb', script)
+
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<worker 1>
+  <source>
+    @type single
+    @id single
+    @label @dummydata
+  </source>
+</worker>
+<label @dummydata>
+  <match dummy>
+    @type null
+    @id   blackhole
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('worker_section1.conf', conf)
+      assert Dir.exist?(@root_path)
+
+      assert_log_matches(
+        create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
+        "#0 fluentd worker is now running worker=0",
+        "#1 fluentd worker is now running worker=1",
+        '#1 adding source type="single"'
+      )
+    end
+
+    test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<source>
+  @type dummy
+  @id dummy
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<worker 1>
+  <match dummy>
+    @type null
+    @id   blackhole
+    <buffer>
+      @type file
+      path #{File.join(@root_path, "buf", "file.*.log")}
+    </buffer>
+  </match>
+</worker>
+CONF
+      conf_path = create_conf_file('worker_section2.conf', conf)
+      assert_log_matches(
+        create_cmdline(conf_path),
+        "#0 fluentd worker is now running worker=0",
+        "#1 fluentd worker is now running worker=1",
+        '#1 adding match pattern="dummy" type="null"'
+      )
+    end
+
+    test 'success to start workers when configured plugins as a chidren of MultiOutput only for specific worker do not support multi worker configuration' do
+      script = <<-EOC
+require 'fluent/plugin/output'
+module Fluent::Plugin
+  class SingleOutput < Output
+    Fluent::Plugin.register_output('single', self)
+    def multi_workers_ready?
+      false
+    end
+    def write(chunk)
+    end
+  end
+end
+EOC
+      plugin_path = create_plugin_file('out_single.rb', script)
+
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<source>
+  @type dummy
+  @id dummy
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<worker 1>
+  <match dummy>
+    @type copy
+    <store>
+      @type single
+    </store>
+    <store>
+      @type single
+    </store>
+  </match>
+</worker>
+CONF
+      conf_path = create_conf_file('worker_section3.conf', conf)
+      assert_log_matches(
+        create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
+        "#0 fluentd worker is now running worker=0",
+        "#1 fluentd worker is now running worker=1",
+        '#1 adding match pattern="dummy" type="copy"'
+      )
+    end
   end
 end

data/test/config/test_element.rb

@@ -400,4 +400,67 @@ CONF
       end
     end
   end
+
+  sub_test_case '#set_target_worker' do
+    test 'set target_worker_id recursively' do
+      e = element('label', '@mytest', {}, [ element('filter', '**'), element('match', '**', {}, [ element('store'), element('store') ]) ])
+      e.set_target_worker_id(1)
+      assert_equal 1, e.target_worker_id
+      assert_equal 1, e.elements[0].target_worker_id
+      assert_equal 1, e.elements[1].target_worker_id
+      assert_equal 1, e.elements[1].elements[0].target_worker_id
+      assert_equal 1, e.elements[1].elements[1].target_worker_id
+    end
+  end
+
+  sub_test_case '#for_every_workers?' do
+    test 'has target_worker_id' do
+      e = element()
+      e.set_target_worker_id(1)
+      assert_false e.for_every_workers?
+    end
+
+    test "doesn't have target_worker_id" do
+      e = element()
+      assert e.for_every_workers?
+    end
+  end
+
+  sub_test_case '#for_this_workers?' do
+    test 'target_worker_id == current worker_id' do
+      e = element()
+      e.set_target_worker_id(0)
+      assert e.for_this_worker?
+    end
+
+    test 'target_worker_id != current worker_id' do
+      e = element()
+      e.set_target_worker_id(1)
+      assert_false e.for_this_worker?
+    end
+
+    test "doesn't have target_worker_id" do
+      e = element()
+      assert_false e.for_this_worker?
+    end
+  end
+
+  sub_test_case '#for_another_worker?' do
+    test 'target_worker_id == current worker_id' do
+      e = element()
+      e.set_target_worker_id(0)
+      assert_false e.for_another_worker?
+    end
+
+    test 'target_worker_id != current worker_id' do
+      e = element()
+      e.set_target_worker_id(1)
+      assert e.for_another_worker?
+    end
+
+    test "doesn't have target_worker_id" do
+      e = element()
+      assert_false e.for_another_worker?
+    end
+  end
 end