fluentd 0.10.57 → 0.10.58

data/lib/fluent/plugin/in_syslog.rb

@@ -84,9 +84,9 @@ module Fluent
     def configure(conf)
       super
 
-      parser = TextParser.new
-      if parser.configure(conf, false)
-        @parser = parser
+      if conf.has_key?('format')
+        @parser = Plugin.new_parser(conf['format'])
+        @parser.configure(conf)
       else
         conf['with_priority'] = true
         @parser = TextParser::SyslogParser.new
@@ -157,7 +157,7 @@ module Fluent
     end
 
     def receive_data(data, addr)
-      @parser.call(data) { |time, record|
+      @parser.parse(data) { |time, record|
         unless time && record
           log.warn "invalid syslog message", :data => data
           return
@@ -192,7 +192,7 @@ module Fluent
 
       tag = "#{@tag}.#{facility}.#{priority}"
 
-      Engine.emit(tag, time, record)
+      router.emit(tag, time, record)
     rescue => e
       log.error "syslog failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
     end

data/lib/fluent/plugin/in_tail.rb

@@ -59,7 +59,7 @@ module Fluent
     end
 
     def configure_parser(conf)
-      @parser = TextParser.new
+      @parser = Plugin.new_parser(conf['format'])
       @parser.configure(conf)
     end
 
@@ -200,7 +200,7 @@ module Fluent
                   else
                     @tag
                   end
-            Engine.emit(tag, time, record)
+            router.emit(tag, time, record)
           else
             log.warn "got incomplete line at shutdown from #{tw.path}: #{lb.inspect}"
           end
@@ -224,7 +224,7 @@ module Fluent
                 @tag
               end
         begin
-          Engine.emit_stream(tag, es)
+          router.emit_stream(tag, es)
         rescue
           # ignore errors. Engine shows logs and backtraces.
         end
@@ -258,9 +258,9 @@ module Fluent
     def parse_multilines(lines, tail_watcher)
       lb = tail_watcher.line_buffer
       es = MultiEventStream.new
-      if @parser.parser.has_firstline?
+      if @parser.has_firstline?
         lines.each { |line|
-          if @parser.parser.firstline?(line)
+          if @parser.firstline?(line)
             if lb
               convert_line_to_event(lb, es)
             end
@@ -790,7 +790,7 @@ module Fluent
 
       unless es.empty?
         begin
-          Engine.emit_stream(@tag, es)
+          router.emit_stream(@tag, es)
         rescue
           # ignore errors. Engine shows logs and backtraces.
         end

data/lib/fluent/plugin/out_copy.rb

@@ -33,7 +33,7 @@ module Fluent
       conf.elements.select {|e|
         e.name == 'store'
       }.each {|e|
-        type = e['type']
+        type = e['@type'] || e['type']
         unless type
           raise ConfigError, "Missing 'type' parameter on <store> directive"
         end

data/lib/fluent/plugin/out_exec_filter.rb

@@ -353,7 +353,7 @@ module Fluent
         tag = @tag
       end
 
-      Engine.emit(tag, time, record)
+      router.emit(tag, time, record)
 
     rescue
       if @suppress_error_log_interval == 0 || Time.now.to_i > @next_log_time

data/lib/fluent/plugin/out_file.rb

@@ -62,8 +62,8 @@ module Fluent
 
       super
 
-      conf['format'] = @format
-      @formatter = TextFormatter.create(conf)
+      @formatter = Plugin.new_formatter(@format)
+      @formatter.configure(conf)
 
       @buffer.symlink_path = @symlink_path if @symlink_path
     end
@@ -74,7 +74,7 @@ module Fluent
 
     def write(chunk)
       path = generate_path(chunk)
-      FileUtils.mkdir_p File.dirname(path)
+      FileUtils.mkdir_p File.dirname(path), :mode => DEFAULT_DIR_PERMISSION
 
       case @compress
       when nil

data/lib/fluent/plugin/out_roundrobin.rb

@@ -35,7 +35,7 @@ module Fluent
       conf.elements.select {|e|
         e.name == 'store'
       }.each {|e|
-        type = e['type']
+        type = e['@type'] || e['type']
         unless type
           raise ConfigError, "Missing 'type' parameter on <store> directive"
         end

data/lib/fluent/plugin/socket_util.rb

@@ -87,7 +87,7 @@ module Fluent
       def configure(conf)
         super
 
-        @parser = TextParser.new
+        @parser = Plugin.new_parser(@format)
         @parser.configure(conf)
       end
 
@@ -130,7 +130,7 @@ module Fluent
           end
 
           record[@source_host_key] = addr[3] if @source_host_key
-          Engine.emit(@tag, time, record)
+          router.emit(@tag, time, record)
         }
       rescue => e
         log.error msg.dump, :error => e, :error_class => e.class, :host => addr[3]

data/lib/fluent/supervisor.rb

@@ -73,6 +73,11 @@ module Fluent
         end
         self
       end
+
+      def level=(level)
+        @level = level
+        $log.level = level
+      end
     end
 
     def self.default_options
@@ -107,8 +112,6 @@ module Fluent
       @chgroup = opt[:chgroup]
       @chuser = opt[:chuser]
 
-      apply_system_config(opt)
-
       @log_level = opt[:log_level]
       @suppress_interval = opt[:suppress_interval]
       @suppress_config_dump = opt[:suppress_config_dump]
@@ -122,6 +125,8 @@ module Fluent
 
     def start
       @log.init
+      read_config
+      apply_system_config
 
       dry_run if @dry_run
       start_daemonize if @daemonize
@@ -129,7 +134,6 @@ module Fluent
         install_supervisor_signal_handlers
         until @finished
           supervise do
-            read_config
             change_privilege
             init_engine
             install_main_process_signal_handlers
@@ -143,7 +147,6 @@ module Fluent
       else
         $log.info "starting fluentd-#{Fluent::VERSION} without supervision"
         main_process do
-          read_config
           change_privilege
           init_engine
           install_main_process_signal_handlers
@@ -169,7 +172,6 @@ module Fluent
     def dry_run
       $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode"
 
-      read_config
       change_privilege
       init_engine
       install_main_process_signal_handlers
@@ -347,6 +349,7 @@ module Fluent
       elsif @inline_config
         @config_data << "\n" << @inline_config.gsub("\\n","\n")
       end
+      @conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
     end
 
     class SystemConfig
@@ -365,37 +368,31 @@ module Fluent
         configure(conf)
       end
 
-      def to_opt
-        opt = {}
-        opt[:log_level] = @log_level unless @log_level.nil?
-        opt[:suppress_interval] = @emit_error_log_interval unless @emit_error_log_interval.nil?
-        opt[:suppress_config_dump] = @suppress_config_dump unless @suppress_config_dump.nil?
-        opt[:suppress_repeated_stacktrace] = @suppress_repeated_stacktrace unless @suppress_repeated_stacktrace.nil?
-        opt[:without_source] = @without_source unless @without_source.nil?
-        opt
+      def apply(supervisor)
+        system = self
+        supervisor.instance_eval {
+          @log.level = @log_level = system.log_level unless system.log_level.nil?
+          @suppress_interval = system.emit_error_log_interval unless system.emit_error_log_interval.nil?
+          @suppress_config_dump = system.suppress_config_dump unless system.suppress_config_dump.nil?
+          @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
+          @without_source = system.without_source unless system.without_source.nil?
+        }
       end
     end
 
     # TODO: this method should be moved to SystemConfig class method
-    def apply_system_config(opt)
-      # Create NULL file to avoid $log uninitialized problem before call @log.init
-      file = File.open(File::NULL)
-      $log = Fluent::Log.new(file, Log::LEVEL_INFO)
-      read_config
-      systems = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config).elements.select { |e|
+    def apply_system_config
+      systems = @conf.elements.select { |e|
         e.name == 'system'
       }
       return if systems.empty?
       raise ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
 
-      opt.merge!(SystemConfig.new(systems.first).to_opt)
-    ensure
-      file.close
+      SystemConfig.new(systems.first).apply(self)
     end
 
     def run_configure
-      conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
-      Fluent::Engine.run_configure(conf)
+      Fluent::Engine.run_configure(@conf)
     end
 
     def change_privilege

data/lib/fluent/test/base.rb

@@ -17,6 +17,20 @@
 #
 module Fluent
   module Test
+    def self.setup
+      Fluent.__send__(:remove_const, :Engine)
+      engine = Fluent.const_set(:Engine, EngineClass.new).init
+
+      engine.define_singleton_method(:now=) {|n|
+        @now = n.to_i
+      }
+      engine.define_singleton_method(:now) {
+        @now || super()
+      }
+
+      nil
+    end
+
     class TestDriver
       include ::Test::Unit::Assertions
 

data/lib/fluent/test/output_test.rb

@@ -78,7 +78,13 @@ module Fluent
             assert_equal(@expected_buffer, buffer)
           end
 
-          chunk = MemoryBufferChunk.new('', buffer)
+          key = ''
+          if @instance.respond_to?(:time_slicer)
+            # this block is only for test_out_file
+            time, record = @entries.first
+            key = @instance.time_slicer.call(time)
+          end
+          chunk = MemoryBufferChunk.new(key, buffer)
           result = @instance.write(chunk)
         }
         result

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.57'
+  VERSION = '0.10.58'
 
 end

data/test/config/test_config_parser.rb

@@ -1,6 +1,6 @@
-require 'helper'
+require_relative '../helper'
+require_relative "assertions"
 require "json"
-require "config/assertions"
 require "fluent/config/error"
 require "fluent/config/basic_parser"
 require "fluent/config/literal_parser"
@@ -229,6 +229,10 @@ module Fluent::Config
         "ignores spacing around element argument" => [root(e("test", "a")), %[
           <test    a    >
           </test>
+        ]],
+        "accepts spacing inside element argument (for multiple tags)" => [root(e("test", "a.** b.**")), %[
+          <test    a.** b.** >
+          </test>
         ]])
       def test_parse_element(data)
         expected, target = data

data/test/config/test_configurable.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/configurable'
 require 'fluent/config/element'
 require 'fluent/config/section'

data/test/config/test_configure_proxy.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/config/configure_proxy'
 
 module Fluent::Config

data/test/config/test_dsl.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/config/element'
 require "fluent/config/dsl"
 

data/test/config/test_literal_parser.rb

@@ -1,5 +1,5 @@
-require "helper"
-require 'config/assertions'
+require_relative "../helper"
+require_relative 'assertions'
 require "fluent/config/error"
 require "fluent/config/literal_parser"
 require "fluent/config/v1_parser"

data/test/config/test_section.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/config/section'
 
 module Fluent::Config

data/test/config/test_system_config.rb

@@ -1,49 +1,99 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/configurable'
 require 'fluent/config/element'
 require 'fluent/config/section'
 require 'fluent/supervisor'
 
 module Fluent::Config
+  class FakeLoggerInitializer
+    attr_accessor :level
+    def initalize
+      @level = nil
+    end
+  end
+
+  class FakeSupervisor
+    def initialize
+      @log = FakeLoggerInitializer.new
+      @log_level = nil
+      @suppress_interval = nil
+      @suppress_config_dump = nil
+      @suppress_repeated_stacktrace = nil
+      @without_source = nil
+    end
+  end
+
   class TestSystemConfig < ::Test::Unit::TestCase
+
     def parse_text(text)
       basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
       Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
     end
 
     test 'should not override default configurations when no parameters' do
-      conf = parse_text(<<EOS)
-<system>
-</system>
-EOS
+      conf = parse_text(<<-EOS)
+        <system>
+        </system>
+      EOS
+      s = FakeSupervisor.new
       sc = Fluent::Supervisor::SystemConfig.new(conf)
+      sc.apply(s)
       assert_nil(sc.log_level)
       assert_nil(sc.suppress_repeated_stacktrace)
       assert_nil(sc.emit_error_log_interval)
       assert_nil(sc.suppress_config_dump)
       assert_nil(sc.without_source)
-      assert_empty(sc.to_opt)
+      assert_nil(s.instance_variable_get(:@log_level))
+      assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
+      assert_nil(s.instance_variable_get(:@emit_error_log_interval))
+      assert_nil(s.instance_variable_get(:@suppress_config_dump))
+      assert_nil(s.instance_variable_get(:@without_source))
     end
 
-    {'log_level' => 'error', 'suppress_repeated_stacktrace' => true, 'emit_error_log_interval' => 60, 'suppress_config_dump' => true, 'without_source' => true}.each { |k, v|
+    {'log_level' => 'error',
+     'suppress_repeated_stacktrace' => true,
+     'emit_error_log_interval' => 60,
+     'suppress_config_dump' => true,
+     'without_source' => true,
+    }.each { |k, v|
       test "accepts #{k} parameter" do
-        conf = parse_text(<<EOS)
-<system>
-  #{k} #{v}
-</system>
-EOS
+        conf = parse_text(<<-EOS)
+          <system>
+            #{k} #{v}
+          </system>
+        EOS
+        s = FakeSupervisor.new
         sc = Fluent::Supervisor::SystemConfig.new(conf)
+        sc.apply(s)
         assert_not_nil(sc.instance_variable_get("@#{k}"))
-        key = (k == 'emit_error_log_interval' ? :suppress_interval : k.to_sym)
-        assert_include(sc.to_opt, key)
+        key = (k == 'emit_error_log_interval' ? 'suppress_interval' : k)
+        assert_not_nil(s.instance_variable_get("@#{key}"))
       end
     }
 
     {'foo' => 'bar', 'hoge' => 'fuga'}.each { |k, v|
       test "should not affect settable parameters with unknown #{k} parameter" do
+        s = FakeSupervisor.new
         sc = Fluent::Supervisor::SystemConfig.new({k => v})
-        assert_empty(sc.to_opt)
+        sc.apply(s)
+        assert_nil(s.instance_variable_get(:@log_level))
+        assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
+        assert_nil(s.instance_variable_get(:@emit_error_log_interval))
+        assert_nil(s.instance_variable_get(:@suppress_config_dump))
+        assert_nil(s.instance_variable_get(:@without_source))
       end
     }
+
+    test 'log_level' do
+      conf = parse_text(<<-EOS)
+        <system>
+          log_level warn
+        </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::Supervisor::SystemConfig.new(conf)
+      sc.apply(s)
+      assert_equal(Fluent::Log::LEVEL_WARN, s.instance_variable_get("@log").level)
+    end
   end
 end