fluentd 1.12.4 → 1.13.0

data/lib/fluent/plugin/out_forward.rb

@@ -227,31 +227,14 @@ module Fluent::Plugin
         socket_cache: socket_cache,
       )
 
-      configs = []
-
-      # rewrite for using server as sd_static
-      conf.elements(name: 'server').each do |s|
-        s.name = 'service'
-      end
-
-      unless conf.elements(name: 'service').empty?
-        # To copy `services` element only
-        new_elem = Fluent::Config::Element.new('static_service_discovery', {}, {}, conf.elements(name: 'service'))
-        configs << { type: :static, conf: new_elem }
-      end
-
-      conf.elements(name: 'service_discovery').each_with_index do |c, i|
-        configs << { type: @service_discovery[i][:@type], conf: c }
-      end
-
-      service_discovery_create_manager(
+      service_discovery_configure(
         :out_forward_service_discovery_watcher,
-        configurations: configs,
+        static_default_service_directive: 'server',
         load_balancer: LoadBalancer.new(log),
         custom_build_method: method(:build_node),
       )
 
-      discovery_manager.services.each do |server|
+      service_discovery_services.each do |server|
         # it's only for test
         @nodes << server
         unless @heartbeat_type == :none
@@ -273,7 +256,7 @@ module Fluent::Plugin
         end
       end
 
-      if discovery_manager.services.empty?
+      if service_discovery_services.empty?
         raise Fluent::ConfigError, "forward output plugin requires at least one node is required. Add <server> or <service_discovery>"
       end
 
@@ -306,7 +289,7 @@ module Fluent::Plugin
 
       unless @heartbeat_type == :none
         if @heartbeat_type == :udp
-          @usock = socket_create_udp(discovery_manager.services.first.host, discovery_manager.services.first.port, nonblock: true)
+          @usock = socket_create_udp(service_discovery_services.first.host, service_discovery_services.first.port, nonblock: true)
           server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length, &method(:on_udp_heatbeat_response_recv))
         end
         timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_heartbeat_timer))
@@ -318,7 +301,7 @@ module Fluent::Plugin
       end
 
       if @verify_connection_at_startup
-        discovery_manager.services.each do |node|
+        service_discovery_services.each do |node|
           begin
             node.verify_connection
           rescue StandardError => e
@@ -374,7 +357,7 @@ module Fluent::Plugin
       return if chunk.empty?
       tag = chunk.metadata.tag
 
-      discovery_manager.select_service { |node| node.send_data(tag, chunk) }
+      service_discovery_select_service { |node| node.send_data(tag, chunk) }
     end
 
     def try_write(chunk)
@@ -384,7 +367,7 @@ module Fluent::Plugin
         return
       end
       tag = chunk.metadata.tag
-      discovery_manager.select_service { |node| node.send_data(tag, chunk) }
+      service_discovery_select_service { |node| node.send_data(tag, chunk) }
       last_ack if @require_ack_response && @suspend_flush
     end
 
@@ -434,7 +417,7 @@ module Fluent::Plugin
 
     def statistics
       stats = super
-      services = discovery_manager.services
+      services = service_discovery_services
       healthy_nodes_count = 0
       registed_nodes_count = services.size
       services.each do |s|
@@ -471,7 +454,7 @@ module Fluent::Plugin
 
     def on_heartbeat_timer
       need_rebuild = false
-      discovery_manager.services.each do |n|
+      service_discovery_services.each do |n|
         begin
           log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
           n.usock = @usock if @usock
@@ -486,16 +469,16 @@ module Fluent::Plugin
       end
 
       if need_rebuild
-        discovery_manager.rebalance
+        service_discovery_rebalance
       end
     end
 
     def on_udp_heatbeat_response_recv(data, sock)
       sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
-      if node = discovery_manager.services.find { |n| n.sockaddr == sockaddr }
+      if node = service_discovery_services.find { |n| n.sockaddr == sockaddr }
         # log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port
         if node.heartbeat
-          discovery_manager.rebalance
+          service_discovery_rebalance
         end
       else
         log.warn("Unknown heartbeat response received from #{sock.remote_host}:#{sock.remote_port}. It may service out")

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -22,10 +22,19 @@ module Fluent
     module ServiceDiscovery
       include Fluent::PluginHelper::Timer
 
+      # For the compatibility with older versions without `param_name: :service_discovery_configs`
+      attr_reader :service_discovery
+
       def self.included(mod)
         mod.include ServiceDiscoveryParams
       end
 
+      def configure(conf)
+        super
+        # For the compatibility with older versions without `param_name: :service_discovery_configs`
+        @service_discovery = @service_discovery_configs
+      end
+
       def start
         unless @discovery_manager
           log.warn('There is no discovery_manager. skip start them')
@@ -52,6 +61,35 @@ module Fluent
 
       private
 
+      # @param title [Symbol] the thread name. this value should be unique.
+      # @param static_default_service_directive [String] the directive name of each service when "static" service discovery is enabled in default
+      # @param load_balancer [Object] object which has two methods #rebalance and #select_service
+      # @param custom_build_method [Proc]
+      def service_discovery_configure(title, static_default_service_directive: nil, load_balancer: nil, custom_build_method: nil, interval: 3)
+        configs = @service_discovery_configs.map(&:corresponding_config_element)
+        if static_default_service_directive
+          configs.prepend Fluent::Config::Element.new(
+            'service_discovery',
+            '',
+            {'@type' => 'static'},
+            @config.elements(name: static_default_service_directive.to_s).map{|e| Fluent::Config::Element.new('service', e.arg, e.dup, e.elements, e.unused) }
+          )
+        end
+        service_discovery_create_manager(title, configurations: configs, load_balancer: load_balancer, custom_build_method: custom_build_method, interval: interval)
+      end
+
+      def service_discovery_select_service(&block)
+        @discovery_manager.select_service(&block)
+      end
+
+      def service_discovery_services
+        @discovery_manager.services
+      end
+
+      def service_discovery_rebalance
+        @discovery_manager.rebalance
+      end
+
       # @param title [Symbol] the thread name. this value should be unique.
       # @param configurations [Hash] hash which must has discivery_service type and its configuration like `{ type: :static, conf: <Fluent::Config::Element> }`
       # @param load_balancer [Object] object which has two methods #rebalance and #select_service
@@ -78,7 +116,7 @@ module Fluent
       module ServiceDiscoveryParams
         include Fluent::Configurable
 
-        config_section :service_discovery do
+        config_section :service_discovery, multi: true, param_name: :service_discovery_configs do
           config_param :@type, :string
         end
       end

data/lib/fluent/plugin_helper/service_discovery/manager.rb

@@ -32,17 +32,23 @@ module Fluent
           @static_config = true
         end
 
-        def configure(opts, parent: nil)
-          opts.each do |opt|
-            sd = Fluent::Plugin.new_sd(opt[:type], parent: parent)
-            sd.configure(opt[:conf])
+        def configure(configs, parent: nil)
+          configs.each do |config|
+            type, conf = if config.has_key?(:conf) # for compatibility with initial API
+                           [config[:type], config[:conf]]
+                         else
+                           [config['@type'], config]
+                         end
+
+            sd = Fluent::Plugin.new_sd(type, parent: parent)
+            sd.configure(conf)
 
             sd.services.each do |s|
               @services[s.discovery_id] = build_service(s)
             end
             @discoveries << sd
 
-            if @static_config && opt[:type] != :static
+            if @static_config && type.to_sym != :static
               @static_config = false
             end
           end

data/lib/fluent/supervisor.rb

@@ -518,6 +518,8 @@ module Fluent
         dl_opts = {}
         # subtract 1 to match serverengine daemon logger side logging severity.
         dl_opts[:log_level] = @level - 1
+        dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
+        dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
         logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
         $log = Fluent::Log.new(logger, @opts)
         $log.enable_color(false) if @path
@@ -606,6 +608,19 @@ module Fluent
 
       @cl_opt = opt
       @conf = nil
+      # parse configuration immediately to initialize logger in early stage
+      if @config_path and File.exist?(@config_path)
+        @conf = Fluent::Config.build(config_path: @config_path,
+                                     encoding: @conf_encoding ? @conf_encoding : 'utf-8',
+                                     additional_config: @inline_config ? @inline_config : nil,
+                                     use_v1_config: !!@use_v1_config)
+        @system_config = build_system_config(@conf)
+        if @system_config.log
+          @log_rotate_age ||= @system_config.log.rotate_age
+          @log_rotate_size ||= @system_config.log.rotate_size
+        end
+        @conf = nil
+      end
 
       log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
                   ignore_same_log_interval: opt[:ignore_same_log_interval]}

data/lib/fluent/system_config.rb

@@ -55,6 +55,20 @@ module Fluent
     config_section :log, required: false, init: true, multi: false do
       config_param :format, :enum, list: [:text, :json], default: :text
       config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
+      config_param :rotate_age, default: nil do |v|
+        if Fluent::Log::LOG_ROTATE_AGE.include?(v)
+          v.to_sym
+        else
+          begin
+            Integer(v)
+          rescue ArgumentError => e
+            raise Fluent::ConfigError, e.message
+          else
+            v.to_i
+          end
+        end
+      end
+      config_param :rotate_size, :size, default: nil
     end
 
     config_section :counter_server, multi: false do

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.12.4'
+  VERSION = '1.13.0'
 
 end

data/test/command/test_cat.rb

@@ -0,0 +1,96 @@
+require_relative '../helper'
+
+require 'test-unit'
+require 'open3'
+require 'fluent/plugin/output'
+require 'fluent/plugin/in_forward'
+require 'fluent/plugin/out_secondary_file'
+require 'fluent/test/driver/output'
+require 'fluent/test/driver/input'
+
+class TestFluentCat < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.mkdir_p(TMP_DIR)
+    @record = { 'key' => 'value' }
+    @time = event_time
+    @es = Fluent::OneEventStream.new(@time, @record)
+    @primary = create_primary
+    metadata = @primary.buffer.new_metadata
+    @chunk = create_chunk(@primary, metadata, @es)
+  end
+
+  def teardown
+    FileUtils.rm_rf(TMP_DIR)
+  end
+
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluent_cat#{ENV['TEST_ENV_NUMBER']}")
+  FLUENT_CAT_COMMAND = File.expand_path(File.dirname(__FILE__) + "/../../bin/fluent-cat")
+
+  PORT = unused_port
+  CONFIG = %[
+    port #{PORT}
+    bind 127.0.0.1
+  ]
+
+  SECONDARY_CONFIG = %[
+    directory #{TMP_DIR}
+  ]
+
+  class DummyOutput < Fluent::Plugin::Output
+    def write(chunk); end
+  end
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput).configure(conf)
+  end
+
+  def create_primary(buffer_cofig = config_element('buffer'))
+    DummyOutput.new.configure(config_element('ROOT','',{}, [buffer_cofig]))
+  end
+
+  def create_secondary_driver(conf=SECONDARY_CONFIG)
+    c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)
+    c.instance.acts_as_secondary(@primary)
+    c.configure(conf)
+  end
+
+  def create_chunk(primary, metadata, es)
+    primary.buffer.generate_chunk(metadata).tap do |c|
+      c.concat(es.to_msgpack_stream, es.size)
+      c.commit
+    end
+  end
+
+  sub_test_case "json" do
+    def test_cat_json
+      d = create_driver
+      d.run(expect_records: 1) do
+        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{PORT} json") do |stdin|
+          stdin.puts('{"key":"value"}')
+          stdin.close
+        end
+      end
+      event = d.events.first
+      assert_equal([1, "json", @record],
+                   [d.events.size, event.first, event.last])
+    end
+  end
+
+  sub_test_case "msgpack" do
+    def test_cat_secondary_file
+      d = create_secondary_driver
+      path = d.instance.write(@chunk)
+      d = create_driver
+      d.run(expect_records: 1) do
+        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{PORT} --format msgpack secondary") do |stdin|
+          stdin.write(File.read(path))
+          stdin.close
+        end
+      end
+      event = d.events.first
+      assert_equal([1, "secondary", @record],
+                   [d.events.size, event.first, event.last])
+    end
+  end
+end

data/test/config/test_system_config.rb

@@ -143,5 +143,51 @@ module Fluent::Config
       sc.overwrite_variables(**s.for_system_config)
       assert_equal(level, sc.log_level)
     end
+
+    sub_test_case "log rotation" do
+      data('daily' => "daily",
+           'weekly' => 'weekly',
+           'monthly' => 'monthly')
+      test "symbols for rotate_age" do |age|
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age #{age}
+            </log>
+          </system>
+        EOS
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(age.to_sym, sc.log.rotate_age)
+      end
+
+      test "numeric number for rotate age" do
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age 3
+            </log>
+          </system>
+        EOS
+        s = FakeSupervisor.new
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(3, sc.log.rotate_age)
+      end
+
+      data(h: ['100', 100],
+           k: ['1k', 1024],
+           m: ['1m', 1024 * 1024],
+           g: ['1g', 1024 * 1024 * 1024])
+      test "numeric and SI prefix for rotate_size" do |(label, size)|
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_size #{label}
+            </log>
+          </system>
+        EOS
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(size, sc.log.rotate_size)
+      end
+    end
   end
 end

data/test/plugin/in_tail/test_io_handler.rb

@@ -30,7 +30,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
     end
 
     returned_lines = ''
-    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: false) do |lines, _watcher|
+    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
       returned_lines << lines.join
       true
     end
@@ -62,7 +62,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = ''
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: true) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true) do |lines, _watcher|
         returned_lines << lines.join
         true
       end
@@ -93,7 +93,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = []
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
         returned_lines << lines.dup
         true
       end
@@ -119,7 +119,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = []
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
         returned_lines << lines.dup
         true
       end