fluentd 0.10.62 → 0.12.0.pre.1

data/lib/fluent/status.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class StatusClass
     def initialize

data/lib/fluent/supervisor.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -73,11 +71,6 @@ module Fluent
         end
         self
       end
-
-      def level=(level)
-        @level = level
-        $log.level = level
-      end
     end
 
     def self.default_options
@@ -94,14 +87,12 @@ module Fluent
         :suppress_interval => 0,
         :suppress_repeated_stacktrace => false,
         :without_source => false,
-        :use_v1_config => false,
-        :supervise => true,
+        :use_v1_config => true,
       }
     end
 
     def initialize(opt)
       @daemonize = opt[:daemonize]
-      @supervise = opt[:supervise]
       @config_path = opt[:config_path]
       @inline_config = opt[:inline_config]
       @use_v1_config = opt[:use_v1_config]
@@ -112,6 +103,8 @@ module Fluent
       @chgroup = opt[:chgroup]
       @chuser = opt[:chuser]
 
+      apply_system_config(opt)
+
       @log_level = opt[:log_level]
       @suppress_interval = opt[:suppress_interval]
       @suppress_config_dump = opt[:suppress_config_dump]
@@ -125,28 +118,13 @@ module Fluent
 
     def start
       @log.init
-      read_config
-      apply_system_config
 
       dry_run if @dry_run
       start_daemonize if @daemonize
-      if @supervise
-        install_supervisor_signal_handlers
-        until @finished
-          supervise do
-            change_privilege
-            init_engine
-            install_main_process_signal_handlers
-            run_configure
-            finish_daemonize if @daemonize
-            run_engine
-            exit 0
-          end
-          $log.error "fluentd main process died unexpectedly. restarting." unless @finished
-        end
-      else
-        $log.info "starting fluentd-#{Fluent::VERSION} without supervision"
-        main_process do
+      install_supervisor_signal_handlers
+      until @finished
+        supervise do
+          read_config
           change_privilege
           init_engine
           install_main_process_signal_handlers
@@ -155,6 +133,7 @@ module Fluent
           run_engine
           exit 0
         end
+        $log.error "fluentd main process died unexpectedly. restarting." unless @finished
       end
     end
 
@@ -172,6 +151,7 @@ module Fluent
     def dry_run
       $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode"
 
+      read_config
       change_privilege
       init_engine
       install_main_process_signal_handlers
@@ -323,16 +303,10 @@ module Fluent
       trap :HUP do
         $log.debug "fluentd supervisor process get SIGHUP"
         $log.info "restarting"
-        # Creating new thread due to mutex can't lock
-        # in main thread during trap context
-        Thread.new {
-          read_config
-          apply_system_config
-          if pid = @main_pid
-            Process.kill(:TERM, pid)
-            # don't resuce Erro::ESRSH here (invalid status)
-          end
-        }.run
+        if pid = @main_pid
+          Process.kill(:TERM, pid)
+          # don't resuce Erro::ESRSH here (invalid status)
+        end
       end
 
       trap :USR1 do
@@ -355,7 +329,6 @@ module Fluent
       elsif @inline_config
         @config_data << "\n" << @inline_config.gsub("\\n","\n")
       end
-      @conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
     end
 
     class SystemConfig
@@ -374,31 +347,37 @@ module Fluent
         configure(conf)
       end
 
-      def apply(supervisor)
-        system = self
-        supervisor.instance_eval {
-          @log.level = @log_level = system.log_level unless system.log_level.nil?
-          @suppress_interval = system.emit_error_log_interval unless system.emit_error_log_interval.nil?
-          @suppress_config_dump = system.suppress_config_dump unless system.suppress_config_dump.nil?
-          @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
-          @without_source = system.without_source unless system.without_source.nil?
-        }
+      def to_opt
+        opt = {}
+        opt[:log_level] = @log_level unless @log_level.nil?
+        opt[:suppress_interval] = @emit_error_log_interval unless @emit_error_log_interval.nil?
+        opt[:suppress_config_dump] = @suppress_config_dump unless @suppress_config_dump.nil?
+        opt[:suppress_repeated_stacktrace] = @suppress_repeated_stacktrace unless @suppress_repeated_stacktrace.nil?
+        opt[:without_source] = @without_source unless @without_source.nil?
+        opt
       end
     end
 
     # TODO: this method should be moved to SystemConfig class method
-    def apply_system_config
-      systems = @conf.elements.select { |e|
+    def apply_system_config(opt)
+      # Create NULL file to avoid $log uninitialized problem before call @log.init
+      file = File.open(File::NULL)
+      $log = Fluent::Log.new(file, Log::LEVEL_INFO)
+      read_config
+      systems = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config).elements.select { |e|
         e.name == 'system'
       }
       return if systems.empty?
       raise ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
 
-      SystemConfig.new(systems.first).apply(self)
+      opt.merge!(SystemConfig.new(systems.first).to_opt)
+    ensure
+      file.close
     end
 
     def run_configure
-      Fluent::Engine.run_configure(@conf)
+      conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
+      Fluent::Engine.run_configure(conf)
     end
 
     def change_privilege

data/lib/fluent/test.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 require 'test/unit'
 require 'fluent/load'
 require 'fluent/test/base'

data/lib/fluent/test/base.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,22 +13,9 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   module Test
-    def self.setup
-      Fluent.__send__(:remove_const, :Engine)
-      engine = Fluent.const_set(:Engine, EngineClass.new).init
-
-      engine.define_singleton_method(:now=) {|n|
-        @now = n.to_i
-      }
-      engine.define_singleton_method(:now) {
-        @now || super()
-      }
-
-      nil
-    end
-
     class TestDriver
       include ::Test::Unit::Assertions
 
@@ -46,6 +31,7 @@ module Fluent
         else
           @instance = klass
         end
+        @instance.router = Engine.root_agent.event_router
         @instance.log = TestLogger.new
 
         @config = Config.new

data/lib/fluent/test/input_test.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class FileBuffer < BasicBuffer
     def self.clear_buffer_paths
@@ -75,19 +74,65 @@ module Fluent
         all
       end
 
+      def register_run_post_condition(&block)
+        if block
+          @run_post_conditions ||= []
+          @run_post_conditions << block
+        end
+      end
+
+      def register_run_breaking_condition(&block)
+        if block
+          @run_breaking_conditions ||= []
+          @run_breaking_conditions << block
+        end
+      end
+
+      def run_should_stop?
+        # Should stop running if post conditions are not registered.
+        return true unless @run_post_conditions
+
+        # Should stop running if all of the post conditions are true.
+        return true if @run_post_conditions.all? {|proc| proc.call }
+
+        # Should stop running if any of the breaking conditions is true.
+        # In this case, some post conditions may be not true.
+        return true if @run_breaking_conditions && @run_breaking_conditions.any? {|proc| proc.call }
+
+        false
+      end
+
       def run(&block)
         m = method(:emit_stream)
         Engine.define_singleton_method(:emit_stream) {|tag,es|
           m.call(tag, es)
         }
+        instance.router.define_singleton_method(:emit_stream) {|tag,es|
+          m.call(tag, es)
+        }
         super {
           block.call if block
 
-          if @expected_emits_length || @expects
-            max_length = @expected_emits_length || @expects.length
-            started_at = Time.now
+          if @expected_emits_length || @expects || @run_post_conditions
+            # counters for emits and emit_streams
             i, j = 0, 0
-            while i < max_length && Time.now <= started_at + @run_timeout
+
+            # Events of expected length will be emitted at the end.
+            max_length = @expected_emits_length
+            max_length ||= @expects.length if @expects
+            if max_length
+              register_run_post_condition do
+                i == max_length
+              end
+            end
+
+            # Set runnning timeout to avoid infinite loop caused by some errors.
+            started_at = Time.now
+            register_run_breaking_condition do
+              Time.now >= started_at + @run_timeout
+            end
+
+            until run_should_stop?
               if j >= @emit_streams.length
                 sleep 0.01
                 next

data/lib/fluent/test/output_test.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   module Test
     class TestOutputChain
@@ -53,10 +52,6 @@ module Fluent
         @entries = []
         @expected_buffer = nil
         @tag = tag
-
-        def @instance.buffer
-          @buffer
-        end
       end
 
       attr_accessor :tag
@@ -82,19 +77,8 @@ module Fluent
             assert_equal(@expected_buffer, buffer)
           end
 
-          key = ''
-          if @instance.respond_to?(:time_slicer)
-            # this block is only for test_out_file
-            time, record = @entries.first
-            key = @instance.time_slicer.call(time)
-          end
-          chunk = @instance.buffer.new_chunk(key)
-          chunk << buffer
-          begin
-            result = @instance.write(chunk)
-          ensure
-            chunk.purge
-          end
+          chunk = MemoryBufferChunk.new('', buffer)
+          result = @instance.write(chunk)
         }
         result
       end

data/lib/fluent/version.rb

@@ -1,5 +1,21 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
 
-  VERSION = '0.10.62'
+  VERSION = '0.12.0.pre.1'
 
 end

data/spec/config/config_parser_spec.rb

@@ -0,0 +1,314 @@
+require "json"
+require "config/helper"
+require "fluent/config/error"
+require "fluent/config/basic_parser"
+require "fluent/config/literal_parser"
+require "fluent/config/v1_parser"
+
+describe Fluent::Config::V1Parser do
+  include_context 'config_helper'
+
+  def read_config(path)
+    path = File.expand_path(path)
+    data = File.read(path)
+    Fluent::Config::V1Parser.parse(data, File.basename(path), File.dirname(path))
+  end
+
+  def parse_text(text)
+    basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+    Fluent::Config::V1Parser.parse(text, '(test)', basepath, nil)
+  end
+
+  def root(*elements)
+    if elements.first.is_a?(Fluent::Config::Element)
+      attrs = {}
+    else
+      attrs = elements.shift || {}
+    end
+    Fluent::Config::Element.new('ROOT', '', attrs, elements)
+  end
+
+  def e(name, arg='', attrs={}, elements=[])
+    Fluent::Config::Element.new(name, arg, attrs, elements)
+  end
+
+  describe 'attribute parsing' do
+    it "parses attributes" do
+      expect(%[
+        k1 v1
+        k2 v2
+      ]).to be_parsed_as(e('ROOT', '', {"k1"=>"v1", "k2"=>"v2"}))
+    end
+
+    it "allows attribute without value" do
+      expect(%[
+        k1
+        k2 v2
+      ]).to be_parsed_as(e('ROOT', '', {"k1"=>"", "k2"=>"v2"}))
+    end
+
+    it "parses attribute key always string" do
+      expect("1 1").to be_parsed_as(e('ROOT', '', {"1" => "1"}))
+    end
+
+    [
+      "_.%$!,",
+      "/=~-~@\`:?",
+      "()*{}.[]",
+    ].each do |v|
+      it "parses a value with symbol #{v.inspect}" do
+        expect("k #{v}").to be_parsed_as(e('ROOT', '', {"k" => v}))
+      end
+    end
+
+    it "ignores spacing around value" do
+      expect("  k1     a    ").to be_parsed_as(e('ROOT', '', {"k1" => "a"}))
+    end
+
+    it "allows spaces in value" do
+      expect("k1 a  b  c").to be_parsed_as(e('ROOT', '', {"k1" => "a  b  c"}))
+    end
+
+    it "ignores comments after value" do
+      expect("  k1 a#comment").to be_parsed_as(e('ROOT', '', {"k1" => "a"}))
+    end
+
+    it "allows # in value if quoted" do
+      expect('  k1 "a#comment"').to be_parsed_as(e('ROOT', '', {"k1" => "a#comment"}))
+    end
+
+    it "rejects characters after quoted string" do
+      expect('  k1 "a" 1').to be_parse_error
+    end
+
+    it "rejects @ prefix in parameter name" do
+      expect('  @k v').to be_parse_error
+    end
+  end
+
+  describe 'element parsing' do
+    it do
+      expect("").to be_parsed_as(root)
+    end
+
+    it "accepts empty element" do
+      expect(%[
+        <test>
+        </test>
+      ]).to be_parsed_as(
+        root(
+          e("test")
+        )
+      )
+    end
+
+    it "accepts argument and attributes" do
+      expect(%[
+        <test var>
+          key val
+        </test>
+      ]).to be_parsed_as(root(
+          e("test", 'var', {'key'=>"val"})
+        ))
+    end
+
+    it "accepts nested elements" do
+      expect(%[
+        <test var>
+          key 1
+          <nested1>
+          </nested1>
+          <nested2>
+          </nested2>
+        </test>
+      ]).to be_parsed_as(root(
+          e("test", 'var', {'key'=>'1'}, [
+            e('nested1'),
+            e('nested2')
+          ])
+        ))
+    end
+
+    it "accepts multiline json values" do
+      expect(%[
+        <test var>
+          key ["a",
+"b", "c",
+"d"]
+        </test>
+      ]).to be_parsed_as(root(
+          e("test", 'var', {'key'=>"[\"a\",\"b\",\"c\",\"d\"]"})
+      ))
+    end
+
+    [
+      "**",
+      "*.*",
+      "1",
+      "_.%$!",
+      "/",
+      "()*{}.[]",
+    ].each do |arg|
+      it "parses element argument #{arg.inspect}" do
+        expect(%[
+          <test #{arg}>
+          </test>
+        ]).to be_parsed_as(root(
+            e("test", arg)
+          ))
+      end
+    end
+
+    it "parses empty element argument to nil" do
+      expect(%[
+        <test >
+        </test>
+      ]).to be_parsed_as(root(
+          e("test", '')
+        ))
+    end
+
+    it "ignores spacing around element argument" do
+      expect(%[
+        <test    a    >
+        </test>
+      ]).to be_parsed_as(root(
+          e("test", "a")
+        ))
+    end
+
+    it "considers comments in element argument" do
+      expect(%[
+        <test #a>
+        </test>
+      ]).to be_parse_error
+    end
+
+    it "requires line_end after begin tag" do
+      expect(%[
+        <test></test>
+      ]).to be_parse_error
+    end
+
+    it "requires line_end after end tag" do
+      expect(%[
+        <test>
+        </test><test>
+        </test>
+      ]).to be_parse_error
+    end
+  end
+
+  # port from test_config.rb
+  describe '@include parsing' do
+    TMP_DIR = File.dirname(__FILE__) + "/tmp/v1_config#{ENV['TEST_ENV_NUMBER']}"
+
+    def write_config(path, data)
+      FileUtils.mkdir_p(File.dirname(path))
+      File.open(path, "w") { |f| f.write data }
+    end
+
+    def prepare_config
+      write_config "#{TMP_DIR}/config_test_1.conf", %[
+        k1 root_config
+        include dir/config_test_2.conf  #
+        @include #{TMP_DIR}/config_test_4.conf
+        include file://#{TMP_DIR}/config_test_5.conf
+        @include config.d/*.conf
+      ]
+      write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+        k2 relative_path_include
+        @include ../config_test_3.conf
+      ]
+      write_config "#{TMP_DIR}/config_test_3.conf", %[
+        k3 relative_include_in_included_file
+      ]
+      write_config "#{TMP_DIR}/config_test_4.conf", %[
+        k4 absolute_path_include
+      ]
+      write_config "#{TMP_DIR}/config_test_5.conf", %[
+        k5 uri_include
+      ]
+      write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+        k6 wildcard_include_1
+        <elem1 name>
+          include normal_parameter
+        </elem1>
+      ]
+      write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+        k7 wildcard_include_2
+      ]
+      write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+        <elem2 name>
+          @include ../dir/config_test_9.conf
+        </elem2>
+      ]
+      write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+        k9 embeded
+        <elem3 name>
+          nested nested_value
+          include hoge
+        </elem3>
+      ]
+      write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+        k8 wildcard_include_3
+        <elem4 name>
+          include normal_parameter
+        </elem4>
+      ]
+    end
+
+    it 'parses @include / include correctly' do
+      prepare_config
+      c = read_config("#{TMP_DIR}/config_test_1.conf")
+      expect(c['k1']).to eq('root_config')
+      expect(c['k2']).to eq('relative_path_include')
+      expect(c['k3']).to eq('relative_include_in_included_file')
+      expect(c['k4']).to eq('absolute_path_include')
+      expect(c['k5']).to eq('uri_include')
+      expect(c['k6']).to eq('wildcard_include_1')
+      expect(c['k7']).to eq('wildcard_include_2')
+      expect(c['k8']).to eq('wildcard_include_3')
+      expect(c.keys).to eq([
+        'k1',
+        'k2',
+        'k3',
+        'k4',
+        'k5',
+        'k8', # Because of the file name this comes first.
+        'k6',
+        'k7',
+      ])
+
+      elem1 = c.elements.find { |e| e.name == 'elem1' }
+      expect(elem1).to be
+      expect(elem1.arg).to eq('name')
+      expect(elem1['include']).to eq('normal_parameter')
+
+      elem2 = c.elements.find { |e| e.name == 'elem2' }
+      expect(elem2).to be
+      expect(elem2.arg).to eq('name')
+      expect(elem2['k9']).to eq('embeded')
+      expect(elem2.has_key?('include')).to be(false)
+
+      elem3 = elem2.elements.find { |e| e.name == 'elem3' }
+      expect(elem3).to be
+      expect(elem3['nested']).to eq('nested_value')
+      expect(elem3['include']).to eq('hoge')
+    end
+
+    # TODO: Add uri based include spec
+  end
+
+  describe 'unescape parameter' do
+    it 'parses dumpped configuration' do
+      original = %q!a\\\\\n\r\f\b\\'\\"z!
+      expected = %!a\\\n\r\f\b'"z!
+
+      conf = parse_text(%[k1 #{original}])
+      expect(conf['k1']).to eq(expected) # escape check
+      conf2 = parse_text(conf.to_s) # use dumpped configuration to check unescape
+      expect(conf2.elements.first['k1']).to eq(expected)
+    end
+  end
+end