fluentd 0.12.0.pre.2 → 0.12.0.pre.3

data/lib/fluent/plugin/socket_util.rb

@@ -104,7 +104,7 @@ module Fluent
       def configure(conf)
         super
 
-        @parser = TextParser.new
+        @parser = Plugin.new_parser(@format)
         @parser.configure(conf)
       end
 
@@ -132,7 +132,7 @@ module Fluent
       private
 
       def on_message(msg, addr)
-        @parser.parse(msg) { |time, record|
+        @parser.call(msg) { |time, record|
           unless time && record
             log.warn "pattern not match: #{msg.inspect}"
             return

data/lib/fluent/registry.rb

@@ -57,34 +57,16 @@ module Fluent
         return
       end
 
-      # search gems
-      if defined?(::Gem::Specification) && ::Gem::Specification.respond_to?(:find_all)
-        specs = Gem::Specification.find_all { |spec|
-          spec.contains_requirable_file? path
-        }
-
-        # prefer newer version
-        specs = specs.sort_by { |spec| spec.version }
-        if spec = specs.last
-          spec.require_paths.each { |lib|
-            file = "#{spec.full_gem_path}/#{lib}/#{path}"
-            require file
-          }
-        end
+      specs = Gem::Specification.find_all { |spec|
+        spec.contains_requirable_file? path
+      }
 
-        # backward compatibility for rubygems < 1.8
-      elsif defined?(::Gem) && ::Gem.respond_to?(:searcher)
-        #files = Gem.find_files(path).sort
-        specs = Gem.searcher.find_all(path)
-
-        # prefer newer version
-        specs = specs.sort_by { |spec| spec.version }
-        specs.reverse_each { |spec|
-          files = Gem.searcher.matching_files(spec, path)
-          unless files.empty?
-            require files.first
-            break
-          end
+      # prefer newer version
+      specs = specs.sort_by { |spec| spec.version }
+      if spec = specs.last
+        spec.require_paths.each { |lib|
+          file = "#{spec.full_gem_path}/#{lib}/#{path}"
+          require file
         }
       end
     end

data/lib/fluent/root_agent.rb

@@ -73,7 +73,7 @@ module Fluent
         if name == ERROR_LABEL
           error_label_config = e
         else
-          add_label(name, e)
+          add_label(name)
           label_configs[name] = e
         end
       }
@@ -96,7 +96,7 @@ module Fluent
 
     def setup_error_label(e)
       # initialize built-in ERROR label
-      error_label = add_label(ERROR_LABEL, e)
+      error_label = add_label(ERROR_LABEL)
       error_label.configure(e)
       error_label.root_agent = RootAgentProxyWithoutErrorCollector.new(self)
       @error_collector = error_label.event_router
@@ -154,7 +154,7 @@ module Fluent
       input
     end
 
-    def add_label(name, e)
+    def add_label(name)
       label = Label.new(name)
       label.root_agent = self
       @labels[name] = label
@@ -168,20 +168,34 @@ module Fluent
       end
     end
 
-    def handle_emits_error(tag, es, e)
+    def emit_error_event(tag, time, record, error)
+      error_info = {:error_class => error.class, :error => error.to_s, :tag => tag, :time => time}
       if @error_collector
+        # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
+        log.warn "send an error event to @ERROR:", error_info
+        @error_collector.emit(tag, time, record)
+      else
+        error_info[:record] = record
+        log.warn "dump an error event:", error_info
+      end
+    end
+
+    def handle_emits_error(tag, es, error)
+      error_info = {:error_class => error.class, :error => error.to_s, :tag => tag}
+      if @error_collector
+        log.warn "send an error event stream to @ERROR:", error_info
         @error_collector.emit_stream(tag, es)
       else
         now = Engine.now
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
-          log.warn "emit transaction failed ", :error_class => e.class, :error => e
+          log.warn "emit transaction failed:", error_info
           log.warn_backtrace
           # log.debug "current next_emit_error_log_time: #{Time.at(@next_emit_error_log_time)}"
           @next_emit_error_log_time = now + @suppress_emit_error_log_interval
           # log.debug "next emit failure log suppressed"
           # log.debug "next logged time is #{Time.at(@next_emit_error_log_time)}"
         end
-        raise e
+        raise error
       end
     end
 
@@ -198,10 +212,15 @@ module Fluent
         suppress_interval(interval_time) unless interval_time.zero?
       end
 
+      def emit_error_event(tag, time, record, error)
+        error_info = {:error_class => error.class, :error => error.to_s, :tag => tag, :time => time, :record => record}
+        log.warn "dump an error event in @ERROR:", error_info
+      end
+
       def handle_emits_error(tag, es, e)
         now = Engine.now
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
-          log.warn "emit transaction failed ", :error_class => e.class, :error => e
+          log.warn "emit transaction failed in @ERROR:", :error_class => e.class, :error => e
           log.warn_backtrace
           @next_emit_error_log_time = now + @suppress_emit_error_log_interval
         end

data/lib/fluent/supervisor.rb

@@ -71,6 +71,11 @@ module Fluent
         end
         self
       end
+
+      def level=(level)
+        @level = level
+        $log.level = level
+      end
     end
 
     def self.default_options
@@ -88,11 +93,13 @@ module Fluent
         :suppress_repeated_stacktrace => true,
         :without_source => false,
         :use_v1_config => true,
+        :supervise => true,
       }
     end
 
     def initialize(opt)
       @daemonize = opt[:daemonize]
+      @supervise = opt[:supervise]
       @config_path = opt[:config_path]
       @inline_config = opt[:inline_config]
       @use_v1_config = opt[:use_v1_config]
@@ -103,8 +110,6 @@ module Fluent
       @chgroup = opt[:chgroup]
       @chuser = opt[:chuser]
 
-      apply_system_config(opt)
-
       @log_level = opt[:log_level]
       @suppress_interval = opt[:suppress_interval]
       @suppress_config_dump = opt[:suppress_config_dump]
@@ -118,13 +123,28 @@ module Fluent
 
     def start
       @log.init
+      read_config
+      apply_system_config
 
       dry_run if @dry_run
       start_daemonize if @daemonize
-      install_supervisor_signal_handlers
-      until @finished
-        supervise do
-          read_config
+      if @supervise
+        install_supervisor_signal_handlers
+        until @finished
+          supervise do
+            change_privilege
+            init_engine
+            install_main_process_signal_handlers
+            run_configure
+            finish_daemonize if @daemonize
+            run_engine
+            exit 0
+          end
+          $log.error "fluentd main process died unexpectedly. restarting." unless @finished
+        end
+      else
+        $log.info "starting fluentd-#{Fluent::VERSION} without supervision"
+        main_process do
           change_privilege
           init_engine
           install_main_process_signal_handlers
@@ -133,7 +153,6 @@ module Fluent
           run_engine
           exit 0
         end
-        $log.error "fluentd main process died unexpectedly. restarting." unless @finished
       end
     end
 
@@ -151,7 +170,6 @@ module Fluent
     def dry_run
       $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode"
 
-      read_config
       change_privilege
       init_engine
       install_main_process_signal_handlers
@@ -329,6 +347,7 @@ module Fluent
       elsif @inline_config
         @config_data << "\n" << @inline_config.gsub("\\n","\n")
       end
+      @conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
     end
 
     class SystemConfig
@@ -347,37 +366,31 @@ module Fluent
         configure(conf)
       end
 
-      def to_opt
-        opt = {}
-        opt[:log_level] = @log_level unless @log_level.nil?
-        opt[:suppress_interval] = @emit_error_log_interval unless @emit_error_log_interval.nil?
-        opt[:suppress_config_dump] = @suppress_config_dump unless @suppress_config_dump.nil?
-        opt[:suppress_repeated_stacktrace] = @suppress_repeated_stacktrace unless @suppress_repeated_stacktrace.nil?
-        opt[:without_source] = @without_source unless @without_source.nil?
-        opt
+      def apply(supervisor)
+        system = self
+        supervisor.instance_eval {
+          @log.level = @log_level = system.log_level unless system.log_level.nil?
+          @suppress_interval = system.emit_error_log_interval unless system.emit_error_log_interval.nil?
+          @suppress_config_dump = system.suppress_config_dump unless system.suppress_config_dump.nil?
+          @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
+          @without_source = system.without_source unless system.without_source.nil?
+        }
       end
     end
 
     # TODO: this method should be moved to SystemConfig class method
-    def apply_system_config(opt)
-      # Create NULL file to avoid $log uninitialized problem before call @log.init
-      file = File.open(File::NULL)
-      $log = Fluent::Log.new(file, Log::LEVEL_INFO)
-      read_config
-      systems = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config).elements.select { |e|
+    def apply_system_config
+      systems = @conf.elements.select { |e|
         e.name == 'system'
       }
       return if systems.empty?
       raise ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
 
-      opt.merge!(SystemConfig.new(systems.first).to_opt)
-    ensure
-      file.close
+      SystemConfig.new(systems.first).apply(self)
     end
 
     def run_configure
-      conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
-      Fluent::Engine.run_configure(conf)
+      Fluent::Engine.run_configure(@conf)
     end
 
     def change_privilege

data/lib/fluent/test.rb

@@ -19,5 +19,6 @@ require 'fluent/load'
 require 'fluent/test/base'
 require 'fluent/test/input_test'
 require 'fluent/test/output_test'
+require 'fluent/test/filter_test'
 
 $log ||= Fluent::Log.new(Fluent::Test::DummyLogDevice.new)

data/lib/fluent/test/base.rb

@@ -16,6 +16,20 @@
 
 module Fluent
   module Test
+    def self.setup
+      Fluent.__send__(:remove_const, :Engine)
+      engine = Fluent.const_set(:Engine, EngineClass.new).init
+
+      engine.define_singleton_method(:now=) {|n|
+        @now = n.to_i
+      }
+      engine.define_singleton_method(:now) {
+        @now || super()
+      }
+
+      nil
+    end
+
     class TestDriver
       include ::Test::Unit::Assertions
 

data/lib/fluent/test/filter_test.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Test
+    class FilterTestDriver < TestDriver
+      def initialize(klass, &block)
+        super(klass, &block)
+      end
+
+      def filter(tag, time, record)
+        @instance.filter(tag, time, record)
+      end
+
+      def filter_stream(tag, es)
+        @instance.filter_stream(tag, es)
+      end
+    end
+  end
+end

data/lib/fluent/test/output_test.rb

@@ -77,7 +77,13 @@ module Fluent
             assert_equal(@expected_buffer, buffer)
           end
 
-          chunk = MemoryBufferChunk.new('', buffer)
+          key = ''
+          if @instance.respond_to?(:time_slicer)
+            # this block is only for test_out_file
+            time, record = @entries.first
+            key = @instance.time_slicer.call(time)
+          end
+          chunk = MemoryBufferChunk.new(key, buffer)
           result = @instance.write(chunk)
         }
         result

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.0.pre.2'
+  VERSION = '0.12.0.pre.3'
 
 end

data/test/config/test_config_parser.rb

@@ -1,6 +1,6 @@
-require 'helper'
+require_relative '../helper'
+require_relative "assertions"
 require "json"
-require "config/assertions"
 require "fluent/config/error"
 require "fluent/config/basic_parser"
 require "fluent/config/literal_parser"
@@ -229,6 +229,10 @@ module Fluent::Config
         "ignores spacing around element argument" => [root(e("test", "a")), %[
           <test    a    >
           </test>
+        ]],
+        "accepts spacing inside element argument (for multiple tags)" => [root(e("test", "a.** b.**")), %[
+          <test    a.** b.** >
+          </test>
         ]])
       def test_parse_element(data)
         expected, target = data

data/test/config/test_configurable.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/configurable'
 require 'fluent/config/element'
 require 'fluent/config/section'

data/test/config/test_configure_proxy.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/config/configure_proxy'
 
 module Fluent::Config

data/test/config/test_dsl.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/config/element'
 require "fluent/config/dsl"
 

data/test/config/test_literal_parser.rb

@@ -1,5 +1,5 @@
-require "helper"
-require 'config/assertions'
+require_relative "../helper"
+require_relative 'assertions'
 require "fluent/config/error"
 require "fluent/config/literal_parser"
 require "fluent/config/v1_parser"

data/test/config/test_section.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/config/section'
 
 module Fluent::Config

data/test/config/test_system_config.rb

@@ -1,49 +1,99 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/configurable'
 require 'fluent/config/element'
 require 'fluent/config/section'
 require 'fluent/supervisor'
 
 module Fluent::Config
+  class FakeLoggerInitializer
+    attr_accessor :level
+    def initalize
+      @level = nil
+    end
+  end
+
+  class FakeSupervisor
+    def initialize
+      @log = FakeLoggerInitializer.new
+      @log_level = nil
+      @suppress_interval = nil
+      @suppress_config_dump = nil
+      @suppress_repeated_stacktrace = nil
+      @without_source = nil
+    end
+  end
+
   class TestSystemConfig < ::Test::Unit::TestCase
+
     def parse_text(text)
       basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
       Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
     end
 
     test 'should not override default configurations when no parameters' do
-      conf = parse_text(<<EOS)
-<system>
-</system>
-EOS
+      conf = parse_text(<<-EOS)
+        <system>
+        </system>
+      EOS
+      s = FakeSupervisor.new
       sc = Fluent::Supervisor::SystemConfig.new(conf)
+      sc.apply(s)
       assert_nil(sc.log_level)
       assert_nil(sc.suppress_repeated_stacktrace)
       assert_nil(sc.emit_error_log_interval)
       assert_nil(sc.suppress_config_dump)
       assert_nil(sc.without_source)
-      assert_empty(sc.to_opt)
+      assert_nil(s.instance_variable_get(:@log_level))
+      assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
+      assert_nil(s.instance_variable_get(:@emit_error_log_interval))
+      assert_nil(s.instance_variable_get(:@suppress_config_dump))
+      assert_nil(s.instance_variable_get(:@without_source))
     end
 
-    {'log_level' => 'error', 'suppress_repeated_stacktrace' => true, 'emit_error_log_interval' => 60, 'suppress_config_dump' => true, 'without_source' => true}.each { |k, v|
+    {'log_level' => 'error',
+     'suppress_repeated_stacktrace' => true,
+     'emit_error_log_interval' => 60,
+     'suppress_config_dump' => true,
+     'without_source' => true,
+    }.each { |k, v|
       test "accepts #{k} parameter" do
-        conf = parse_text(<<EOS)
-<system>
-  #{k} #{v}
-</system>
-EOS
+        conf = parse_text(<<-EOS)
+          <system>
+            #{k} #{v}
+          </system>
+        EOS
+        s = FakeSupervisor.new
         sc = Fluent::Supervisor::SystemConfig.new(conf)
+        sc.apply(s)
         assert_not_nil(sc.instance_variable_get("@#{k}"))
-        key = (k == 'emit_error_log_interval' ? :suppress_interval : k.to_sym)
-        assert_include(sc.to_opt, key)
+        key = (k == 'emit_error_log_interval' ? 'suppress_interval' : k)
+        assert_not_nil(s.instance_variable_get("@#{key}"))
       end
     }
 
     {'foo' => 'bar', 'hoge' => 'fuga'}.each { |k, v|
       test "should not affect settable parameters with unknown #{k} parameter" do
+        s = FakeSupervisor.new
         sc = Fluent::Supervisor::SystemConfig.new({k => v})
-        assert_empty(sc.to_opt)
+        sc.apply(s)
+        assert_nil(s.instance_variable_get(:@log_level))
+        assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
+        assert_nil(s.instance_variable_get(:@emit_error_log_interval))
+        assert_nil(s.instance_variable_get(:@suppress_config_dump))
+        assert_nil(s.instance_variable_get(:@without_source))
       end
     }
+
+    test 'log_level' do
+      conf = parse_text(<<-EOS)
+        <system>
+          log_level warn
+        </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::Supervisor::SystemConfig.new(conf)
+      sc.apply(s)
+      assert_equal(Fluent::Log::LEVEL_WARN, s.instance_variable_get("@log").level)
+    end
   end
 end