fluentd 0.14.9 → 0.14.10

data/lib/fluent/plugin_helper/socket.rb

@@ -0,0 +1,101 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'socket'
+require 'ipaddr'
+
+require_relative 'socket_option'
+
+module Fluent
+  module PluginHelper
+    module Socket
+      # stop     : [-]
+      # shutdown : [-]
+      # close    : [-]
+      # terminate: [-]
+
+      include Fluent::PluginHelper::SocketOption
+
+      attr_reader :_sockets # for tests
+
+      # TODO: implement connection pool for specified host
+
+      def socket_create(proto, host, port, **kwargs, &block)
+        case proto
+        when :tcp
+          socket_create_tcp(host, port, **kwargs, &block)
+        when :udp
+          socket_create_udp(host, port, **kwargs, &block)
+        when :tls
+          socket_create_tls(host, port, **kwargs, &block)
+        when :unix
+          raise "not implemented yet"
+        else
+          raise ArgumentError, "invalid protocol: #{proto}"
+        end
+      end
+
+      def socket_create_tcp(host, port, resolve_name: false, **kwargs, &block)
+        sock = TCPSocket.new(host, port)
+        socket_option_set(sock, resolve_name: resolve_name, **kwargs)
+        if block
+          begin
+            block.call(sock)
+          ensure
+            sock.close_write rescue nil
+            sock.close rescue nil
+          end
+        else
+          sock
+        end
+      end
+
+      def socket_create_udp(host, port, resolve_name: false, connect: false, **kwargs, &block)
+        family = IPAddr.new(IPSocket.getaddress(host)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
+        sock = UDPSocket.new(family)
+        socket_option_set(sock, resolve_name: resolve_name, **kwargs)
+        sock.connect(host, port) if connect
+        if block
+          begin
+            block.call(sock)
+          ensure
+            sock.close rescue nil
+          end
+        else
+          sock
+        end
+      end
+
+      def socket_create_tls(host, port, resolve_name: false, certopts: {}, &block)
+        raise "not implemented yet"
+      end
+
+      # socket_create_socks ?
+
+      def initialize
+        super
+        # @_sockets = [] # for keepalived sockets / connection pool
+      end
+
+      # def close
+      #   @_sockets.each do |sock|
+      #     sock.close
+      #   end
+      #   super
+      # end
+    end
+  end
+end

data/lib/fluent/plugin_helper/socket_option.rb

@@ -0,0 +1,84 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'socket'
+require 'fcntl'
+
+# this module is only for Socket/Server plugin helpers
+module Fluent
+  module PluginHelper
+    module SocketOption
+      FORMAT_STRUCT_LINGER  = 'I!I!' # { int l_onoff; int l_linger; }
+      FORMAT_STRUCT_TIMEVAL = 'L!L!' # { time_t tv_sec; suseconds_t tv_usec; }
+
+      def socket_option_validate!(protocol, resolve_name: nil, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, certopts: nil)
+        unless resolve_name.nil?
+          if protocol != :tcp && protocol != :udp && protocol != :tls
+            raise ArgumentError, "BUG: resolve_name in available for tcp/udp/tls"
+          end
+        end
+        if linger_timeout
+          if protocol != :tcp && protocol != :tls
+            raise ArgumentError, "BUG: linger_timeout is available for tcp/tls"
+          end
+        end
+        if certopts
+          if protocol != :tls
+            raise ArgumentError, "BUG: certopts is available only for tls"
+          end
+        else
+          if protocol == :tls
+            raise ArgumentError, "BUG: certopts (certificate options) not specified for TLS"
+            socket_option_certopts_validate!(certopts)
+          end
+        end
+      end
+
+      def socket_option_certopts_validate!(certopts)
+        raise "not implemented yet"
+      end
+
+      def socket_option_set(sock, resolve_name: nil, nonblock: false, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, certopts: nil)
+        unless resolve_name.nil?
+          sock.do_not_reverse_lookup = !resolve_name
+        end
+        if nonblock
+          sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
+        end
+        if linger_timeout
+          optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
+          socket_option_set_one(sock, :SO_LINGER, optval)
+        end
+        if recv_timeout
+          optval = [recv_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)
+          socket_option_set_one(sock, :SO_RCVTIMEO, optval)
+        end
+        if send_timeout
+          optval = [send_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)
+          socket_option_set_one(sock, :SO_SNDTIMEO, optval)
+        end
+        # TODO: certopts for TLS
+        sock
+      end
+
+      def socket_option_set_one(sock, option, value)
+        sock.setsockopt(::Socket::SOL_SOCKET, option, value)
+      rescue => e
+        log.warn "failed to set socket option", sock: sock.class, option: option, value: value, error: e
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/timer.rb

@@ -36,6 +36,7 @@ module Fluent
         timer = TimerWatcher.new(title, interval, repeat, log, checker, &block)
         @_timers << title
         event_loop_attach(timer)
+        timer
       end
 
       def timer_running?

data/lib/fluent/test/driver/base.rb

@@ -18,6 +18,8 @@ require 'fluent/config'
 require 'fluent/config/element'
 require 'fluent/log'
 
+require 'serverengine/socket_manager'
+require 'fileutils'
 require 'timeout'
 
 module Fluent
@@ -32,20 +34,18 @@ module Fluent
 
         def initialize(klass, opts: {}, &block)
           if klass.is_a?(Class)
+            @instance = klass.new
             if block
-              # Create new class for test w/ overwritten methods
-              #   klass.dup is worse because its ancestors does NOT include original class name
-              klass_name = klass.name
-              klass = Class.new(klass)
-              klass.define_singleton_method("name") { klass_name }
-              klass.module_eval(&block)
+              @instance.singleton_class.module_eval(&block)
+              @instance.send(:initialize)
             end
-            @instance = klass.new
           else
             @instance = klass
           end
           @instance.under_plugin_development = true
 
+          @socket_manager_server = nil
+
           @logs = []
 
           @test_clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
@@ -101,6 +101,15 @@ module Fluent
         end
 
         def instance_start
+          if @instance.respond_to?(:server_wait_until_start)
+            @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+            if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+              FileUtils.rm_f @socket_manager_path
+            end
+            @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
+            ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+          end
+
           unless @instance.started?
             @instance.start
           end
@@ -130,22 +139,45 @@ module Fluent
         def instance_shutdown
           instance_hook_before_stopped
 
-          @instance.stop            unless @instance.stopped?
-          @instance.before_shutdown unless @instance.before_shutdown?
-          @instance.shutdown        unless @instance.shutdown?
+          unless @instance.stopped?
+            @instance.stop rescue nil
+          end
+          unless @instance.before_shutdown?
+            @instance.before_shutdown rescue nil
+          end
+          unless @instance.shutdown?
+            @instance.shutdown rescue nil
+          end
 
           if @instance.respond_to?(:event_loop_wait_until_stop)
             @instance.event_loop_wait_until_stop
           end
 
-          @instance.after_shutdown  unless @instance.after_shutdown?
-          @instance.close     unless @instance.closed?
+          unless @instance.after_shutdown?
+            @instance.after_shutdown rescue nil
+          end
+          unless @instance.closed?
+            @instance.close rescue nil
+          end
 
           if @instance.respond_to?(:thread_wait_until_stop)
             @instance.thread_wait_until_stop
           end
 
-          @instance.terminate unless @instance.terminated?
+          if @instance.respond_to?(:server_wait_until_stop)
+            @instance.server_wait_until_stop
+          end
+
+          unless @instance.terminated?
+            @instance.terminate rescue nil
+          end
+
+          if @socket_manager_server
+            @socket_manager_server.close
+            if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+              FileUtils.rm_f @socket_manager_path
+            end
+          end
         end
 
         def run_actual(timeout: DEFAULT_TIMEOUT, &block)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.9'
+  VERSION = '0.14.10'
 
 end

data/lib/fluent/winsvc.rb

@@ -58,7 +58,7 @@ begin
       ev.set
       ev.close
       if @pid > 0
-        Porcess.waitpid(@pid)
+        Process.waitpid(@pid)
       end
     end
   end

data/test/compat/test_parser.rb

@@ -79,4 +79,14 @@ class TextParserTest < ::Test::Unit::TestCase
     p2.configure('format' => 'none')
     assert_equal false, p2.parser.estimate_current_event
   end
+
+  data(ignorecase: Regexp::IGNORECASE,
+       multiline: Regexp::MULTILINE,
+       both: Regexp::IGNORECASE & Regexp::MULTILINE)
+  def test_regexp_parser_config(options)
+    source = "a"
+    parser = Fluent::TextParser::RegexpParser.new(Regexp.new(source, options), { "dummy" => "dummy" })
+    regexp = parser.instance_variable_get("@regexp")
+    assert_equal(options, regexp.options)
+  end
 end

data/test/config/test_configurable.rb

@@ -1295,6 +1295,26 @@ module Fluent::Config
         assert_raise Fluent::ObsoletedParameterError.new("'key2' parameter is already removed: key2 has been removed.") do
           obj.configure(config_element('ROOT', '', {'key2' => 'yay'}, []))
         end
+        first_log = obj.log.logs.first
+        assert{ first_log && first_log.include?("[error]") && first_log.include?("config error in:\n<ROOT>\n  key2 yay\n</ROOT>") }
+      end
+
+      sub_test_case 'logger is nil' do
+        test 'nothing raised if deprecated parameter is configured' do
+          obj = ConfigurableSpec::UnRecommended.new
+          obj.log = nil
+          obj.configure(config_element('ROOT', '', {'key1' => 'yay'}, []))
+          assert_nil(obj.log)
+        end
+
+        test 'NoMethodError is not raised if obsoleted parameter is configured' do
+          obj = ConfigurableSpec::UnRecommended.new
+          obj.log = nil
+          assert_raise Fluent::ObsoletedParameterError.new("'key2' parameter is already removed: key2 has been removed.") do
+            obj.configure(config_element('ROOT', '', {'key2' => 'yay'}, []))
+          end
+          assert_nil(obj.log)
+        end
       end
     end
 

data/test/helper.rb

@@ -69,7 +69,18 @@ end
 
 include Fluent::Test::Helpers
 
-def unused_port(num = 1)
+def unused_port(num = 1, protocol: :tcp, bind: "0.0.0.0")
+  case protocol
+  when :tcp
+    unused_port_tcp(num)
+  when :udp
+    unused_port_udp(num, bind: bind)
+  else
+    raise ArgumentError, "unknown protocol: #{protocol}"
+  end
+end
+
+def unused_port_tcp(num = 1)
   ports = []
   sockets = []
   num.times do
@@ -85,6 +96,30 @@ def unused_port(num = 1)
   end
 end
 
+PORT_RANGE_AVAILABLE = (1024...65535)
+
+def unused_port_udp(num = 1, bind: "0.0.0.0")
+  family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
+  ports = []
+  sockets = []
+  while ports.size < num
+    port = rand(PORT_RANGE_AVAILABLE)
+    u = UDPSocket.new(family)
+    if (u.bind(bind, port) rescue nil)
+      ports << port
+      sockets << u
+    else
+      u.close
+    end
+  end
+  sockets.each{|s| s.close }
+  if num == 1
+    return ports.first
+  else
+    return *ports
+  end
+end
+
 def waiting(seconds, logs: nil, plugin: nil)
   begin
     Timeout.timeout(seconds) do

data/test/plugin/test_filter_record_transformer.rb

@@ -2,9 +2,11 @@ require_relative '../helper'
 require 'timecop'
 require 'fluent/test/driver/filter'
 require 'fluent/plugin/filter_record_transformer'
+require 'flexmock/test_unit'
 
 class RecordTransformerFilterTest < Test::Unit::TestCase
   include Fluent
+  include FlexMock::TestCase
 
   setup do
     Test.setup
@@ -53,7 +55,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         hostname ${hostname}
         tag ${tag}
         time ${time}
-        message ${hostname} ${tag_parts[-1]} ${message}
+        message ${hostname} ${tag_parts[-1]} ${record["message"]}
       </record>
     ]
 
@@ -119,7 +121,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       config = %[
         enable_ruby yes
         <record>
-          message ${hostname} ${tag_parts.last} ${"'" + message + "'"}
+          message ${hostname} ${tag_parts.last} ${"'" + record["message"] + "'"}
         </record>
       ]
       msgs = ['1', '2']
@@ -263,8 +265,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           enable_ruby #{enable_ruby}
           remove_keys eventType0
           <record>
-            message bar ${message}
-            eventtype ${eventType0}
+            message bar ${record["message"]}
+            eventtype ${record["eventType0"]}
           </record>
         ]
         msgs = ['1', '2']
@@ -360,11 +362,11 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           auto_typecast false
           enable_ruby #{enable_ruby}
           <record>
-            single      ${source}
-            multiple    ${source}${source}
-            with_prefix prefix-${source}
-            with_suffix ${source}-suffix
-            with_quote  source[""]
+            single      ${record["source"]}
+            multiple    ${record["source"]}${record["source"]}
+            with_prefix prefix-${record["source"]}
+            with_suffix ${record["source"]}-suffix
+            with_quote  record["source"][""]
           </record>
         ]
         msgs = [
@@ -379,27 +381,27 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             multiple: "stringstring",
             with_prefix: "prefix-string",
             with_suffix: "string-suffix",
-            with_quote: %Q{source[""]} },
+            with_quote: %Q{record["source"][""]} },
           { single: 123.to_s,
             multiple: "#{123.to_s}#{123.to_s}",
             with_prefix: "prefix-#{123.to_s}",
             with_suffix: "#{123.to_s}-suffix",
-            with_quote: %Q{source[""]} },
+            with_quote: %Q{record["source"][""]} },
           { single: [1, 2].to_s,
             multiple: "#{[1, 2].to_s}#{[1, 2].to_s}",
             with_prefix: "prefix-#{[1, 2].to_s}",
             with_suffix: "#{[1, 2].to_s}-suffix",
-            with_quote: %Q{source[""]} },
+            with_quote: %Q{record["source"][""]} },
           { single: {a:1, b:2}.to_s,
             multiple: "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
             with_prefix: "prefix-#{{a:1, b:2}.to_s}",
             with_suffix: "#{{a:1, b:2}.to_s}-suffix",
-            with_quote: %Q{source[""]} },
+            with_quote: %Q{record["source"][""]} },
           { single: nil.to_s,
             multiple: "#{nil.to_s}#{nil.to_s}",
             with_prefix: "prefix-#{nil.to_s}",
             with_suffix: "#{nil.to_s}-suffix",
-            with_quote: %Q{source[""]} },
+            with_quote: %Q{record["source"][""]} },
         ]
         actual_results = []
         filtered = filter(config, msgs)
@@ -420,10 +422,10 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           auto_typecast yes
           enable_ruby #{enable_ruby}
           <record>
-            single      ${source}
-            multiple    ${source}${source}
-            with_prefix prefix-${source}
-            with_suffix ${source}-suffix
+            single      ${record["source"]}
+            multiple    ${record["source"]}${record["source"]}
+            with_prefix prefix-${record["source"]}
+            with_suffix ${record["source"]}-suffix
           </record>
         ]
         msgs = [
@@ -467,28 +469,6 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         end
         assert_equal(expected_results, actual_results)
       end
-
-      test %Q[record["key"] with enable_ruby #{enable_ruby}] do
-        config = %[
-          enable_ruby #{enable_ruby}
-          auto_typecast yes
-          <record>
-            _timestamp ${record["@timestamp"]}
-            _foo_bar   ${record["foo.bar"]}
-          </record>
-        ]
-        d = create_driver(config)
-        record = {
-          "foo.bar"    => "foo.bar",
-          "@timestamp" => 10,
-        }
-        d.run { d.feed(@tag, @time, record) }
-        filtered = d.filtered
-        filtered.each do |t, r|
-          assert { r['_timestamp'] == record['@timestamp'] }
-          assert { r['_foo_bar'] == record['foo.bar'] }
-        end
-      end
     end
 
     test 'unknown placeholder (enable_ruby no)' do
@@ -503,30 +483,11 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       }
     end
 
-    data(auto_typecast_yes: ["yes", "unknown['bar']"],
-         auto_typecast_no: ["no", "%Q[\#{unknown['bar']}]"])
-    test 'failed to expand (enable_ruby yes)' do |(param, expected_log)|
-
-      config = %[
-        enable_ruby yes
-        auto_typecast #{param}
-        <record>
-          message ${unknown['bar']}
-        </record>
-      ]
-      filtered = filter(config) { |d|
-        mock(d.instance.log).warn("failed to expand `#{expected_log}`", anything)
-      }
-      filtered.each do |t, r|
-        assert_nil(r['message'])
-      end
-    end
-
     test 'expand fields starting with @ (enable_ruby no)' do
       config = %[
         enable_ruby no
         <record>
-          foo ${@timestamp}
+          foo ${record["@timestamp"]}
         </record>
       ]
       d = create_driver(config)
@@ -555,54 +516,21 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         assert_equal([message["@timestamp"]], r['foo'])
       end
     end
+  end # test placeholders
 
-    test 'expand fields starting with @ (enable_ruby yes)' do
-      config = %[
+  sub_test_case 'test error record' do
+    test 'invalid record for placeholders' do
+      d = create_driver(%[
         enable_ruby yes
         <record>
-          foo ${__send__("@timestamp")}
+          foo ${record["unknown"]["key"]}
         </record>
-      ]
-      d = create_driver(config)
-      message = {"@timestamp" => "foo"}
-      d.run { d.feed(@tag, @time, message) }
-      filtered = d.filtered
-      filtered.each do |t, r|
-        assert_equal(message["@timestamp"], r['foo'])
+      ])
+      flexmock(d.instance.router).should_receive(:emit_error_event).
+        with(String, Fluent::EventTime, Hash, RuntimeError).once
+      d.run do
+        d.feed(@tag, Fluent::EventTime.now, {'key' => 'value'})
       end
     end
-  end # test placeholders
-
-  test "compatibility test (enable_ruby yes)" do
-    config = %[
-      enable_ruby yes
-      auto_typecast yes
-      <record>
-        _message   prefix-${message}-suffix
-        _time      ${Time.at(time)}
-        _number    ${number == '-' ? 0 : number}
-        _match     ${/0x[0-9a-f]+/.match(hex)[0]}
-        _timestamp ${__send__("@timestamp")}
-        _foo_bar   ${__send__('foo.bar')}
-      </record>
-    ]
-    d = create_driver(config)
-    record = {
-      "number"     => "-",
-      "hex"        => "0x10",
-      "foo.bar"    => "foo.bar",
-      "@timestamp" => 10,
-      "message"    => "10",
-    }
-    d.run { d.feed(@tag, @time, record) }
-    filtered = d.filtered
-    filtered.each do |t, r|
-      assert { r['_message'] == "prefix-#{record['message']}-suffix" }
-      assert { r['_time'] == Time.at(@time) }
-      assert { r['_number'] == 0 }
-      assert { r['_match'] == record['hex'] }
-      assert { r['_timestamp'] == record['@timestamp'] }
-      assert { r['_foo_bar'] == record['foo.bar'] }
-    end
   end
 end