fluentd 1.5.2 → 1.6.0

data/lib/fluent/plugin_helper/http_server/request.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async/http/protocol'
+require 'fluent/plugin_helper/http_server/methods'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class Request
+        attr_reader :path, :query_string
+
+        def initialize(request)
+          @request = request
+          path = request.path
+          @path, @query_string = path.split('?', 2)
+        end
+
+        def query
+          @query_string && CGI.parse(@query_string)
+        end
+
+        def body
+          @request.body && @request.body.read
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/router.rb

@@ -0,0 +1,54 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async/http/protocol'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class Router
+        class NotFoundApp
+          def self.call(req)
+            [404, { 'Content-Type' => 'text/plain' }, "404 Not Found: #{req.path}\n"]
+          end
+        end
+
+        def initialize(default_app = nil)
+          @router = { get: {}, head: {}, post: {}, put: {}, patch: {}, delete: {}, connect: {}, options: {}, trace: {} }
+          @default_app = default_app || NotFoundApp
+        end
+
+        # @param method [Symbol]
+        # @param path [String]
+        # @param app [Object]
+        def mount(method, path, app)
+          if @router[method].include?(path)
+            raise "#{path} is already mounted"
+          end
+
+          @router[method][path] = app
+        end
+
+        # @param method [Symbol]
+        # @param path [String]
+        # @param request [Fluent::PluginHelper::HttpServer::Request]
+        def route!(method, path, request)
+          @router.fetch(method).fetch(path, @default_app).call(request)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/server.rb

@@ -0,0 +1,87 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async'
+require 'async/http'
+require 'async/http/endpoint'
+
+require 'fluent/plugin_helper/http_server/app'
+require 'fluent/plugin_helper/http_server/router'
+require 'fluent/plugin_helper/http_server/methods'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class Server
+        # @param default_app [Object] This method must have #call.
+        def initialize(addr:, port:, logger:, default_app: nil)
+          @addr = addr
+          @port = port
+          @logger = logger
+
+          # TODO: support https and http2
+          @uri = URI("http://#{@addr}:#{@port}").to_s
+          @router = Router.new(default_app)
+          @reactor = Async::Reactor.new
+          @server = Async::HTTP::Server.new(
+            App.new(@router, @logger),
+            Async::HTTP::Endpoint.parse(@uri)
+          )
+
+          if block_given?
+            yield(self)
+          end
+        end
+
+        def start(notify = nil)
+          @logger.debug("Start async HTTP server listening #{@uri}")
+          task = @reactor.run do
+            @server.run
+
+            if notify
+              notify.push(:ready)
+            end
+          end
+
+          task.stop
+          @logger.debug('Finished HTTP server')
+        end
+
+        def stop
+          @logger.debug('closing HTTP server')
+
+          if @reactor
+            @reactor.stop
+          end
+        end
+
+        HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
+          define_method(name) do |path, app = nil, &block|
+            unless path.end_with?('/')
+              path << '/'
+            end
+
+            if (block && app) || (!block && !app)
+              raise 'You must specify either app or block in the same time'
+            end
+
+            @router.mount(name, path, app || block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/socket.rb

@@ -55,8 +55,14 @@ module Fluent
         end
       end
 
-      def socket_create_tcp(host, port, resolve_name: false, **kwargs, &block)
-        sock = WrappedSocket::TCP.new(host, port)
+      def socket_create_tcp(host, port, resolve_name: false, connect_timeout: nil, **kwargs, &block)
+        sock = if connect_timeout
+                 s = ::Socket.tcp(host, port, connect_timeout: connect_timeout)
+                 s.autoclose = false # avoid GC triggered close
+                 WrappedSocket::TCP.for_fd(s.fileno)
+               else
+                 WrappedSocket::TCP.new(host, port)
+               end
         socket_option_set(sock, resolve_name: resolve_name, **kwargs)
         if block
           begin

data/lib/fluent/supervisor.rb

@@ -249,7 +249,7 @@ module Fluent
       config_fname = File.basename(path)
       config_basedir = File.dirname(path)
       # Assume fluent.conf encoding is UTF-8
-      config_data = File.open(path, "r:utf-8:utf-8") {|f| f.read }
+      config_data = File.open(path, "r:#{params['conf_encoding']}:utf-8") {|f| f.read }
       inline_config = params['inline_config']
       if inline_config == '-'
         config_data << "\n" << STDIN.read
@@ -422,6 +422,7 @@ module Fluent
         supervise: true,
         standalone_worker: false,
         signame: nil,
+        conf_encoding: 'utf-8'
       }
     end
 
@@ -440,6 +441,7 @@ module Fluent
       @config_path = opt[:config_path]
       @inline_config = opt[:inline_config]
       @use_v1_config = opt[:use_v1_config]
+      @conf_encoding = opt[:conf_encoding]
       @log_path = opt[:log_path]
       @dry_run = opt[:dry_run]
       @show_plugin_config = opt[:show_plugin_config]
@@ -618,6 +620,7 @@ module Fluent
       params['chuser'] = @chuser
       params['chgroup'] = @chgroup
       params['use_v1_config'] = @use_v1_config
+      params['conf_encoding'] = @conf_encoding
 
       # system config parameters
       params['workers'] = @workers
@@ -763,7 +766,7 @@ module Fluent
     def read_config
       @config_fname = File.basename(@config_path)
       @config_basedir = File.dirname(@config_path)
-      @config_data = File.open(@config_path, "r:utf-8:utf-8") {|f| f.read }
+      @config_data = File.open(@config_path, "r:#{@conf_encoding}:utf-8") {|f| f.read }
       if @inline_config == '-'
         @config_data << "\n" << STDIN.read
       elsif @inline_config

data/lib/fluent/time.rb

@@ -69,6 +69,19 @@ module Fluent
       @sec.to_s
     end
 
+    begin
+      # ruby 2.5 or later
+      Time.at(0, 0, :nanosecond)
+
+      def to_time
+        Time.at(@sec, @nsec, :nanosecond)
+      end
+    rescue
+      def to_time
+        Time.at(@sec, @nsec / 1000.0)
+      end
+    end
+
     def to_json(*args)
       @sec.to_s
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.5.2'
+  VERSION = '1.6.0'
 
 end

data/test/command/test_fluentd.rb

@@ -28,9 +28,9 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     end
   end
 
-  def create_conf_file(name, content)
+  def create_conf_file(name, content, ext_enc = 'utf-8')
     conf_path = File.join(TMP_DIR, name)
-    File.open(conf_path, 'w') do |file|
+    File.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
       file.write content
     end
     conf_path
@@ -203,6 +203,40 @@ CONF
     end
   end
 
+  sub_test_case 'with --conf-encoding' do
+    test 'runs successfully' do
+      conf = <<CONF
+# テスト
+<source>
+  @type dummy
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<match dummy>
+  @type null
+</match>
+CONF
+      conf_path = create_conf_file('shift_jis.conf', conf, 'shift_jis')
+      assert_log_matches(create_cmdline(conf_path, '--conf-encoding', 'shift_jis'), "fluentd worker is now running", 'worker=0')
+    end
+
+    test 'failed to run by invalid encoding' do
+      conf = <<CONF
+# テスト
+<source>
+  @type dummy
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<match dummy>
+  @type null
+</match>
+CONF
+      conf_path = create_conf_file('shift_jis.conf', conf, 'shift_jis')
+      assert_fluentd_fails_to_start(create_cmdline(conf_path), "invalid byte sequence in UTF-8")
+    end
+  end
+
   sub_test_case 'with system configuration about root directory' do
     setup do
       @root_path = File.join(TMP_DIR, "rootpath")

data/test/helper.rb

@@ -49,6 +49,7 @@ require 'fluent/plugin_helper'
 require 'fluent/msgpack_factory'
 require 'fluent/time'
 require 'serverengine'
+require 'helpers/fuzzy_assert'
 
 module Fluent
   module Plugin

data/test/helpers/fuzzy_assert.rb

@@ -0,0 +1,89 @@
+require 'test/unit'
+
+class FuzzyIncludeAssertion
+  include Test::Unit::Assertions
+
+  def self.assert(expected, actual, message = nil)
+    new(expected, actual, message).assert
+  end
+
+  def initialize(expected, actual, message)
+    @expected = expected
+    @actual = actual
+    @message = message
+  end
+
+  def assert
+    if collection?
+      assert_same_collection
+    else
+      assert_same_value
+    end
+  end
+
+  private
+
+  def assert_same_value
+    m = "expected(#{@expected}) !== actual(#{@actual.inspect})"
+    if @message
+      m = "#{@message}: #{m}"
+    end
+    assert_true(@expected === @actual, m)
+  end
+
+  def assert_same_class
+    if @expected.class != @actual.class
+     if (@expected.class.ancestors | @actual.class.ancestors).empty?
+       assert_equal(@expected.class, @actual.class, @message)
+     end
+    end
+  end
+
+  def assert_same_collection
+    assert_same_class
+    assert_same_values
+  end
+
+  def assert_same_values
+    if @expected.is_a?(Array)
+      @expected.each_with_index do |val, i|
+        self.class.assert(val, @actual[i], @message)
+      end
+    else
+      @expected.each do |key, val|
+        self.class.assert(val, @actual[key], "#{key}: ")
+      end
+    end
+  end
+
+  def collection?
+    @actual.is_a?(Array) || @actual.is_a?(Hash)
+  end
+end
+
+class FuzzyAssertion < FuzzyIncludeAssertion
+  private
+
+  def assert_same_collection
+    super
+    assert_same_keys
+  end
+
+  def assert_same_keys
+    if @expected.is_a?(Array)
+      assert_equal(@expected.size, @actual.size, "expected.size(#{@expected}) != actual.size(#{@expected})")
+    else
+      assert_equal(@expected.keys.sort, @actual.keys.sort)
+    end
+  end
+end
+
+module FuzzyAssert
+  def assert_fuzzy_include(left, right, message = nil)
+    FuzzyIncludeAssertion.new(left, right, message).assert
+  end
+
+  def assert_fuzzy_equal(left, right, message = nil)
+    FuzzyAssertion.new(left, right, message).assert
+  end
+end

data/test/plugin/test_buf_file.rb

@@ -912,7 +912,7 @@ class FileBufferTest < Test::Unit::TestCase
       assert_equal mode, staged[m].state
     end
 
-     def compare_queued_chunk(queued, id, num, mode)
+    def compare_queued_chunk(queued, id, num, mode)
       assert_equal 1, queued.size
       assert_equal id, queued[0].unique_id
       assert_equal num, queued[0].size

data/test/plugin/test_in_http.rb

@@ -707,7 +707,6 @@ class HttpInputTest < Test::Unit::TestCase
       ["tag2", time, {"a"=>2}],
     ]
     res_codes = []
-    res_headers = []
 
     d.run do
       events.each do |tag, time, record|
@@ -731,7 +730,6 @@ class HttpInputTest < Test::Unit::TestCase
       ["tag2", time, {"a"=>2}],
     ]
     res_codes = []
-    res_headers = []
 
     d.run do
       events.each do |tag, time, record|
@@ -803,10 +801,9 @@ class HttpInputTest < Test::Unit::TestCase
         # Send two requests the second one has no Content-Type in Keep-Alive
         Net::HTTP.start("127.0.0.1", PORT) do |http|
           req = Net::HTTP::Post.new("/foodb/bartbl", {"connection" => "keepalive", "Content-Type" => "application/json"})
-          res = http.request(req)
-
+          http.request(req)
           req = Net::HTTP::Get.new("/foodb/bartbl", {"connection" => "keepalive"})
-          res = http.request(req)
+          http.request(req)
         end
 
       end