fluentd 0.14.0 → 0.14.1

data/lib/fluent/test/driver/owner.rb

@@ -0,0 +1,32 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin_id'
+require 'fluent/log'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Test
+    module Driver
+      class Owner < Fluent::Plugin::Base
+        include PluginId
+        include PluginLoggerMixin
+        include PluginHelper::Mixin
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/parser.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owned'
+
+module Fluent
+  module Test
+    module Driver
+      class Parser < BaseOwned
+        def initialize(klass, **kwargs, &block)
+          super
+          @section_name = "parse"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/helpers.rb

@@ -0,0 +1,54 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/element'
+require 'fluent/msgpack_factory'
+require 'fluent/time'
+
+module Fluent
+  module Test
+    module Helpers
+      def config_element(name = 'test', argument = '', params = {}, elements = [])
+        Fluent::Config::Element.new(name, argument, params, elements)
+      end
+
+      def event_time(str=nil, format: nil)
+        if str
+          if format
+            Fluent::EventTime.from_time(Time.strptime(str, format))
+          else
+            Fluent::EventTime.parse(str)
+          end
+        else
+          Fluent::EventTime.now
+        end
+      end
+
+      def msgpack(type)
+        case type
+        when :factory
+          Fluent::MessagePackFactory.factory
+        when :packer
+          Fluent::MessagePackFactory.packer
+        when :unpacker
+          Fluent::MessagePackFactory.unpacker
+        else
+          raise ArgumentError, "unknown msgpack object type '#{type}'"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/log.rb

@@ -0,0 +1,73 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'serverengine'
+require 'fluent/log'
+
+module Fluent
+  module Test
+    class DummyLogDevice
+      attr_reader :logs
+
+      def initialize
+        @logs = []
+      end
+
+      def reset
+        @logs = []
+      end
+
+      def tty?
+        false
+      end
+
+      def puts(*args)
+        args.each{ |arg| write(arg + "\n") }
+      end
+
+      def write(message)
+        @logs.push message
+      end
+
+      def flush
+        true
+      end
+
+      def close
+        true
+      end
+    end
+
+    class TestLogger < Fluent::PluginLogger
+      def initialize
+        @logdev = DummyLogDevice.new
+        dl_opts = {}
+        dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO
+        logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
+        log = Fluent::Log.new(logger)
+        super(log)
+      end
+
+      def reset
+        @logdev.reset
+      end
+
+      def logs
+        @logdev.logs
+      end
+    end
+  end
+end

data/lib/fluent/time.rb

@@ -16,6 +16,7 @@
 
 require 'time'
 require 'msgpack'
+require 'fluent/timezone'
 
 module Fluent
   class EventTime
@@ -101,4 +102,74 @@ module Fluent
       @sec.send(name, *args, &block)
     end
   end
+
+  class TimeFormatter
+    def initialize(format, localtime, timezone = nil)
+      @tc1 = 0
+      @tc1_str = nil
+      @tc2 = 0
+      @tc2_str = nil
+
+      if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/
+        define_singleton_method(:format, method(:format_with_subsec))
+        define_singleton_method(:call, method(:format_with_subsec))
+      else
+        define_singleton_method(:format, method(:format_without_subsec))
+        define_singleton_method(:call, method(:format_with_subsec))
+      end
+
+      formatter = Fluent::Timezone.formatter(timezone, format)
+      @format_nocache = case
+                        when formatter           then formatter
+                        when format && localtime then ->(time){ Time.at(time).strftime(format) }
+                        when format              then ->(time){ Time.at(time).utc.strftime(format) }
+                        when localtime           then ->(time){ Time.at(time).iso8601 }
+                        else                          ->(time){ Time.at(time).utc.iso8601 }
+                        end
+    end
+
+    def format_without_subsec(time)
+      if @tc1 == time
+        return @tc1_str
+      elsif @tc2 == time
+        return @tc2_str
+      else
+        str = format_nocache(time)
+        if @tc1 < @tc2
+          @tc1 = time
+          @tc1_str = str
+        else
+          @tc2 = time
+          @tc2_str = str
+        end
+        return str
+      end
+    end
+
+    def format_with_subsec(time)
+      if Fluent::EventTime.eq?(@tc1, time)
+        return @tc1_str
+      elsif Fluent::EventTime.eq?(@tc2, time)
+        return @tc2_str
+      else
+        str = format_nocache(time)
+        if @tc1 < @tc2
+          @tc1 = time
+          @tc1_str = str
+        else
+          @tc2 = time
+          @tc2_str = str
+        end
+        return str
+      end
+    end
+
+    ## Dynamically defined in #initialize
+    # def format(time)
+    # end
+
+    def format_nocache(time)
+      @format_nocache.call(time)
+    end
+  end
 end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.0'
+  VERSION = '0.14.1'
 
 end

data/test/compat/test_parser.rb

@@ -0,0 +1,82 @@
+require_relative '../helper'
+require 'fluent/plugin/parser'
+
+class TextParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  class MultiEventTestParser < ::Fluent::Parser
+    include Fluent::Configurable
+
+    def parse(text)
+      2.times { |i|
+        record = {}
+        record['message'] = text
+        record['number'] = i
+        yield Fluent::Engine.now, record
+      }
+    end
+  end
+
+  Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })
+
+  def test_lookup_unknown_format
+    assert_raise Fluent::ConfigError do
+      Fluent::Plugin.new_parser('unknown')
+    end
+  end
+
+  data('register_formatter' => 'known', 'register_template' => 'known_old')
+  def test_lookup_known_parser(data)
+    $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), '..', 'scripts'))
+    assert_nothing_raised Fluent::ConfigError do
+      Fluent::Plugin.new_parser(data)
+    end
+    $LOAD_PATH.shift
+  end
+
+  def test_parse_with_return
+    parser = Fluent::TextParser.new
+    parser.configure('format' => 'none')
+    _time, record = parser.parse('log message!')
+    assert_equal({'message' => 'log message!'}, record)
+  end
+
+  def test_parse_with_block
+    parser = Fluent::TextParser.new
+    parser.configure('format' => 'none')
+    parser.parse('log message!') { |time, record|
+      assert_equal({'message' => 'log message!'}, record)
+    }
+  end
+
+  def test_multi_event_parser
+    parser = Fluent::TextParser.new
+    parser.configure('format' => 'multi_event_test')
+    i = 0
+    parser.parse('log message!') { |time, record|
+      assert_equal('log message!', record['message'])
+      assert_equal(i, record['number'])
+      i += 1
+    }
+  end
+
+  def test_setting_estimate_current_event_value
+    p1 = Fluent::TextParser.new
+    assert_nil p1.estimate_current_event
+    assert_nil p1.parser
+
+    p1.configure('format' => 'none')
+    assert_equal true, p1.parser.estimate_current_event
+
+    p2 = Fluent::TextParser.new
+    assert_nil p2.estimate_current_event
+    assert_nil p2.parser
+
+    p2.estimate_current_event = false
+
+    p2.configure('format' => 'none')
+    assert_equal false, p2.parser.estimate_current_event
+  end
+end

data/test/config/test_configure_proxy.rb

@@ -17,6 +17,7 @@ module Fluent::Config
           assert_equal(:section, proxy.name)
           assert_nil(proxy.param_name)
           assert_equal(:section, proxy.variable_name)
+          assert_false(proxy.root?)
           assert_nil(proxy.init)
           assert_nil(proxy.required)
           assert_false(proxy.required?)
@@ -94,6 +95,20 @@ module Fluent::Config
           assert_false(proxy.multi?)
           assert_equal :subsection, proxy.configured_in_section
         end
+
+        test "does overwrite name of proxy for root sections which are used for plugins" do
+          # latest plugin class shows actual plugin implementation
+          p1 = Fluent::Config::ConfigureProxy.new('Fluent::Plugin::MyP1'.to_sym, root: true, required: true, multi: false, type_lookup: @type_lookup)
+          p1.config_param :key1, :integer
+
+          p2 = Fluent::Config::ConfigureProxy.new('Fluent::Plugin::MyP2'.to_sym, root: true, required: true, multi: false, type_lookup: @type_lookup)
+          p2.config_param :key2, :string, default: "value2"
+
+          merged = p1.merge(p2)
+
+          assert_equal 'Fluent::Plugin::MyP2'.to_sym, merged.name
+          assert_true merged.root?
+        end
       end
 
       sub_test_case '#overwrite_defaults' do

data/test/config/test_dsl.rb

@@ -2,6 +2,66 @@ require_relative '../helper'
 require 'fluent/config/element'
 require "fluent/config/dsl"
 
+TMP_DIR = File.dirname(__FILE__) + "/tmp/config_dsl#{ENV['TEST_ENV_NUMBER']}"
+def write_config(path, data)
+  FileUtils.mkdir_p(File.dirname(path))
+  File.open(path, "w") {|f| f.write data }
+end
+
+def prepare_config1
+  write_config "#{TMP_DIR}/config_test_1.conf", %[
+  k1 root_config
+  include dir/config_test_2.conf  #
+  @include #{TMP_DIR}/config_test_4.conf
+  include file://#{TMP_DIR}/config_test_5.conf
+  @include config.d/*.conf
+]
+  write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+  k2 relative_path_include
+  @include ../config_test_3.conf
+]
+  write_config "#{TMP_DIR}/config_test_3.conf", %[
+  k3 relative_include_in_included_file
+]
+  write_config "#{TMP_DIR}/config_test_4.conf", %[
+  k4 absolute_path_include
+]
+  write_config "#{TMP_DIR}/config_test_5.conf", %[
+  k5 uri_include
+]
+  write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+  k6 wildcard_include_1
+  <elem1 name>
+    include normal_parameter
+  </elem1>
+]
+  write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+  k7 wildcard_include_2
+]
+  write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+  <elem2 name>
+    @include ../dir/config_test_9.conf
+  </elem2>
+]
+  write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+  k9 embeded
+  <elem3 name>
+    nested nested_value
+    include hoge
+  </elem3>
+]
+  write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+  k8 wildcard_include_3
+  <elem4 name>
+    include normal_parameter
+  </elem4>
+]
+end
+
+def prepare_config2
+  write_config "#{TMP_DIR}/config_test_1.rb", DSL_CONFIG_EXAMPLE
+end
+
 DSL_CONFIG_EXAMPLE = %q[
 worker {
   hostname = "myhostname"
@@ -57,6 +117,14 @@ source {
 }
 ]
 
+DSL_CONFIG_EXAMPLE_FOR_INCLUDE_CONF = %q[
+include "#{TMP_DIR}/config_test_1.conf"
+]
+
+DSL_CONFIG_EXAMPLE_FOR_INCLUDE_RB = %q[
+include "#{TMP_DIR}/config_test_1.rb"
+]
+
 DSL_CONFIG_RETURNS_NON_ELEMENT = %q[
 worker {
 }
@@ -74,6 +142,9 @@ match('aa','bb'){
 DSL_CONFIG_WRONG_SYNTAX3 = %q[
 match('aa','bb')
 ]
+DSL_CONFIG_WRONG_SYNTAX4 = %q[
+include
+]
 
 module Fluent::Config
   class TestDSLParser < ::Test::Unit::TestCase
@@ -158,6 +229,112 @@ module Fluent::Config
       end
     end
 
+    sub_test_case 'with include conf' do
+      def setup
+        prepare_config1
+        @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_FOR_INCLUDE_CONF, 'dsl_config_for_include.conf')
+      end
+      test 'include config' do
+        assert_equal('root_config', @root['k1'])
+        assert_equal('relative_path_include', @root['k2'])
+        assert_equal('relative_include_in_included_file', @root['k3'])
+        assert_equal('absolute_path_include', @root['k4'])
+        assert_equal('uri_include', @root['k5'])
+        assert_equal('wildcard_include_1', @root['k6'])
+        assert_equal('wildcard_include_2', @root['k7'])
+        assert_equal('wildcard_include_3', @root['k8'])
+        assert_equal([
+            'k1',
+            'k2',
+            'k3',
+            'k4',
+            'k5',
+            'k8', # Because of the file name this comes first.
+            'k6',
+            'k7',
+          ], @root.keys)
+
+        elem1 = @root.elements.find { |e| e.name == 'elem1' }
+        assert(elem1)
+        assert_equal('name', elem1.arg)
+        assert_equal('normal_parameter', elem1['include'])
+
+        elem2 = @root.elements.find { |e| e.name == 'elem2' }
+        assert(elem2)
+        assert_equal('name', elem2.arg)
+        assert_equal('embeded', elem2['k9'])
+        assert_not_include(elem2, 'include')
+
+        elem3 = elem2.elements.find { |e| e.name == 'elem3' }
+        assert(elem3)
+        assert_equal('nested_value', elem3['nested'])
+        assert_equal('hoge', elem3['include'])
+      end
+
+      # TODO: Add uri based include spec
+    end
+
+    sub_test_case 'with include rb' do
+      def setup
+        prepare_config2
+        @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_FOR_INCLUDE_RB, 'dsl_config_for_include.rb')
+      end
+      sub_test_case '.parse' do
+        test 'makes root element' do
+          assert_equal('ROOT', @root.name)
+          assert_predicate(@root.arg, :empty?)
+          assert_equal(0, @root.keys.size)
+        end
+
+        test 'makes worker element for worker tag' do
+          assert_equal(1, @root.elements.size)
+
+          worker = @root.elements.first
+          assert_equal('worker', worker.name)
+          assert_predicate(worker.arg, :empty?)
+          assert_equal(0, worker.keys.size)
+          assert_equal(10, worker.elements.size)
+        end
+
+        test 'makes subsections for blocks, with variable substitution' do
+          ele4 = @root.elements.first.elements[4]
+
+          assert_equal('source', ele4.name)
+          assert_predicate(ele4.arg, :empty?)
+          assert_equal(2, ele4.keys.size)
+          assert_equal('tail', ele4['type'])
+          assert_equal("/var/log/httpd/access.part4.log", ele4['path'])
+        end
+
+        test 'makes user-defined sections with blocks' do
+          filter0 = @root.elements.first.elements[4].elements.first
+
+          assert_equal('filter', filter0.name)
+          assert_equal('bar.**', filter0.arg)
+          assert_equal('hoge', filter0['type'])
+          assert_equal('moge', filter0['val1'])
+          assert_equal(JSON.dump(['foo', 'bar', 'baz']), filter0['val2'])
+          assert_equal('10', filter0['val3'])
+          assert_equal('hoge', filter0['id'])
+
+          assert_equal(2, filter0.elements.size)
+          assert_equal('subsection', filter0.elements[0].name)
+          assert_equal('bar', filter0.elements[0]['foo'])
+          assert_equal('subsection', filter0.elements[1].name)
+          assert_equal('baz', filter0.elements[1]['foo'])
+        end
+
+        test 'makes values with user-assigned variable substitutions' do
+          match0 = @root.elements.first.elements[4].elements.last
+
+          assert_equal('match', match0.name)
+          assert_equal('{foo,bar}.**', match0.arg)
+          assert_equal('file', match0['type'])
+          assert_equal('/var/log/httpd/access.myhostname.4.log', match0['path'])
+        end
+      end
+    end
+
     sub_test_case 'with configuration that returns non element on top' do
       sub_test_case '.parse' do
         test 'does not crash' do
@@ -170,8 +347,9 @@ module Fluent::Config
       sub_test_case '.parse' do
         test 'raises ArgumentError correctly' do
           assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX1, 'dsl_config_wrong_syntax1') }
-          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX2, 'dsl_config_wrong_syntax1') }
-          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX3, 'dsl_config_wrong_syntax1') }
+          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX2, 'dsl_config_wrong_syntax2') }
+          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX3, 'dsl_config_wrong_syntax3') }
+          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX4, 'dsl_config_wrong_syntax4') }
         end
       end
     end