fluentd 1.7.4-x64-mingw32 → 1.8.0-x64-mingw32

data/lib/fluent/plugin/formatter_stdout.rb

@@ -28,7 +28,7 @@ module Fluent
       def configure(conf)
         super
 
-        @time_formatter = Strftime.new(TIME_FORMAT)
+        @time_formatter = Strftime.new(@time_format || TIME_FORMAT)
         @sub_formatter = Plugin.new_formatter(@output_type, parent: self.owner)
         @sub_formatter.configure(conf)
       end

data/lib/fluent/plugin/in_forward.rb

@@ -256,7 +256,7 @@ module Fluent::Plugin
             serializer = :to_json.to_proc
             feeder = ->(d){ parser << d }
           else # msgpack
-            parser = Fluent::Engine.msgpack_factory.unpacker
+            parser = Fluent::MessagePackFactory.msgpack_unpacker
             serializer = :to_msgpack.to_proc
             feeder = ->(d){
               parser.feed_each(d){|obj|

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -64,7 +64,8 @@ module Fluent::Plugin
       def config_ltsv(_req)
         obj = {
           'pid' => Process.pid,
-          'ppid' => Process.ppid
+          'ppid' => Process.ppid,
+          'version' => Fluent::VERSION,
         }.merge(@agent.fluentd_opts)
 
         render_ltsv([obj])
@@ -73,7 +74,8 @@ module Fluent::Plugin
       def config_json(req)
         obj = {
           'pid' => Process.pid,
-          'ppid' => Process.ppid
+          'ppid' => Process.ppid,
+          'version' => Fluent::VERSION,
         }.merge(@agent.fluentd_opts)
         opts = build_option(req)
 

data/lib/fluent/plugin/in_tail.rb

@@ -204,6 +204,12 @@ module Fluent::Plugin
       timer_execute(:in_tail_refresh_watchers, @refresh_interval, &method(:refresh_watchers))
     end
 
+    def stop
+      @@pos_file_paths.delete(@pos_file)
+
+      super
+    end
+
     def shutdown
       # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
       stop_watchers(@tails.keys, immediate: true, remove_watcher: false)

data/lib/fluent/plugin/in_unix.rb

@@ -143,7 +143,7 @@ module Fluent
           @y.on_parse_complete = @on_message
         else
           m = method(:on_read_msgpack)
-          @u = Fluent::Engine.msgpack_factory.unpacker
+          @u = Fluent::MessagePackFactory.msgpack_unpacker
         end
 
         singleton_class.module_eval do

data/lib/fluent/plugin/out_forward.rb

@@ -18,6 +18,7 @@ require 'fluent/output'
 require 'fluent/config/error'
 require 'fluent/clock'
 require 'base64'
+require 'forwardable'
 
 require 'fluent/compat/socket_util'
 require 'fluent/plugin/out_forward/handshake_protocol'
@@ -32,7 +33,7 @@ module Fluent::Plugin
   class ForwardOutput < Output
     Fluent::Plugin.register_output('forward', self)
 
-    helpers :socket, :server, :timer, :thread, :compat_parameters
+    helpers :socket, :server, :timer, :thread, :compat_parameters, :service_discovery
 
     LISTEN_PORT = 24224
 
@@ -224,23 +225,41 @@ module Fluent::Plugin
         socket_cache: socket_cache,
       )
 
-      @servers.each do |server|
-        failure = FailureDetector.new(@heartbeat_interval, @hard_timeout, Time.now.to_i.to_f)
-        name = server.name || "#{server.host}:#{server.port}"
+      configs = []
 
-        log.info "adding forwarding server '#{name}'", host: server.host, port: server.port, weight: server.weight, plugin_id: plugin_id
-        if @heartbeat_type == :none
-          @nodes << NoneHeartbeatNode.new(self, server, failure: failure, connection_manager: @connection_manager, ack_handler: @ack_handler)
-        else
-          node = Node.new(self, server, failure: failure, connection_manager: @connection_manager, ack_handler: @ack_handler)
+      # rewrite for using server as sd_static
+      conf.elements(name: 'server').each do |s|
+        s.name = 'service'
+      end
+
+      unless conf.elements(name: 'service').empty?
+        # To copy `services` element only
+        new_elem = Fluent::Config::Element.new('static_service_discovery', {}, {}, conf.elements(name: 'service'))
+        configs << { type: :static, conf: new_elem }
+      end
+
+      conf.elements(name: 'service_discovery').each_with_index do |c, i|
+        configs << { type: @service_discovery[i][:@type], conf: c }
+      end
+
+      service_discovery_create_manager(
+        :out_forward_service_discovery_watcher,
+        configurations: configs,
+        load_balancer: LoadBalancer.new(log),
+        custom_build_method: method(:build_node),
+      )
+
+      discovery_manager.services.each do |server|
+        # it's only for test
+        @nodes << server
+        unless @heartbeat_type == :none
           begin
-            node.validate_host_resolution!
+            server.validate_host_resolution!
           rescue => e
             raise unless @ignore_network_errors_at_startup
             log.warn "failed to resolve node name when configured", server: (server.name || server.host), error: e
-            node.disable!
+            server.disable!
           end
-          @nodes << node
         end
       end
 
@@ -252,8 +271,8 @@ module Fluent::Plugin
         end
       end
 
-      if @nodes.empty?
-        raise Fluent::ConfigError, "forward output plugin requires at least one <server> is required"
+      if discovery_manager.services.empty?
+        raise Fluent::ConfigError, "forward output plugin requires at least one node is required. Add <server> or <service_discovery>"
       end
 
       if !@keepalive && @keepalive_timeout
@@ -274,12 +293,9 @@ module Fluent::Plugin
     def start
       super
 
-      @load_balancer = LoadBalancer.new(log)
-      @load_balancer.rebuild_weight_array(@nodes)
-
       unless @heartbeat_type == :none
         if @heartbeat_type == :udp
-          @usock = socket_create_udp(@nodes.first.host, @nodes.first.port, nonblock: true)
+          @usock = socket_create_udp(discovery_manager.services.first.host, discovery_manager.services.first.port, nonblock: true)
           server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length, &method(:on_udp_heatbeat_response_recv))
         end
         timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_heartbeat_timer))
@@ -297,7 +313,7 @@ module Fluent::Plugin
       end
 
       if @verify_connection_at_startup
-        @nodes.each do |node|
+        discovery_manager.services.each do |node|
           begin
             node.verify_connection
           rescue StandardError => e
@@ -333,7 +349,7 @@ module Fluent::Plugin
       return if chunk.empty?
       tag = chunk.metadata.tag
 
-      @load_balancer.select_healthy_node { |node| node.send_data(tag, chunk) }
+      discovery_manager.select_service { |node| node.send_data(tag, chunk) }
     end
 
     def try_write(chunk)
@@ -343,7 +359,7 @@ module Fluent::Plugin
         return
       end
       tag = chunk.metadata.tag
-      @load_balancer.select_healthy_node { |n| n.send_data(tag, chunk) }
+      discovery_manager.select_service { |node| node.send_data(tag, chunk) }
     end
 
     def create_transfer_socket(host, port, hostname, &block)
@@ -387,6 +403,23 @@ module Fluent::Plugin
       end
     end
 
+    def statistics
+      stats = super
+      services = discovery_manager.services
+      healthy_nodes_count = 0
+      registed_nodes_count = services.size
+      services.each do |s|
+        if s.available?
+          healthy_nodes_count += 1
+        end
+      end
+
+      stats.merge(
+        'healthy_nodes_count' => healthy_nodes_count,
+        'registered_nodes_count' => registed_nodes_count,
+      )
+    end
+
     # MessagePack FixArray length is 3
     FORWARD_HEADER = [0x93].pack('C').freeze
     def forward_header
@@ -395,9 +428,21 @@ module Fluent::Plugin
 
     private
 
+    def build_node(server)
+      name = server.name || "#{server.host}:#{server.port}"
+      log.info "adding forwarding server '#{name}'", host: server.host, port: server.port, weight: server.weight, plugin_id: plugin_id
+
+      failure = FailureDetector.new(@heartbeat_interval, @hard_timeout, Time.now.to_i.to_f)
+      if @heartbeat_type == :none
+        NoneHeartbeatNode.new(self, server, failure: failure, connection_manager: @connection_manager, ack_handler: @ack_handler)
+      else
+        Node.new(self, server, failure: failure, connection_manager: @connection_manager, ack_handler: @ack_handler)
+      end
+    end
+
     def on_heartbeat_timer
       need_rebuild = false
-      @nodes.each do |n|
+      discovery_manager.services.each do |n|
         begin
           log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
           n.usock = @usock if @usock
@@ -412,19 +457,19 @@ module Fluent::Plugin
       end
 
       if need_rebuild
-        @load_balancer.rebuild_weight_array(@nodes)
+        discovery_manager.rebalance
       end
     end
 
     def on_udp_heatbeat_response_recv(data, sock)
       sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
-      if node = @nodes.find { |n| n.sockaddr == sockaddr }
+      if node = discovery_manager.services.find { |n| n.sockaddr == sockaddr }
         # log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port
         if node.heartbeat
-          @load_balancer.rebuild_weight_array(@nodes)
+          discovery_manager.rebalance
         end
       else
-        log.warn("Unknown heartbeat response received from #{sock.remote_host}:#{sock.remote_port}")
+        log.warn("Unknown heartbeat response received from #{sock.remote_host}:#{sock.remote_port}. It may service out")
       end
     end
 
@@ -463,12 +508,16 @@ module Fluent::Plugin
     end
 
     class Node
+      extend Forwardable
+      def_delegators :@server, :discovery_id, :host, :port, :name, :weight, :standby, :username, :password, :shared_key
+
       # @param connection_manager [Fluent::Plugin::ForwardOutput::ConnectionManager]
       # @param ack_handler [Fluent::Plugin::ForwardOutput::AckHandler]
       def initialize(sender, server, failure:, connection_manager:, ack_handler:)
         @sender = sender
         @log = sender.log
         @compress = sender.compress
+        @server = server
 
         @name = server.name
         @host = server.host
@@ -496,7 +545,7 @@ module Fluent::Plugin
           username: server.username,
         )
 
-        @unpacker = Fluent::Engine.msgpack_unpacker
+        @unpacker = Fluent::MessagePackFactory.msgpack_unpacker
 
         @resolved_host = nil
         @resolved_time = 0
@@ -508,7 +557,7 @@ module Fluent::Plugin
 
       attr_accessor :usock
 
-      attr_reader :name, :host, :port, :weight, :standby, :state
+      attr_reader :state
       attr_reader :sockaddr  # used by on_udp_heatbeat_response_recv
       attr_reader :failure # for test
 

data/lib/fluent/plugin/out_forward/ack_handler.rb

@@ -34,7 +34,7 @@ module Fluent::Plugin
         @timeout = timeout
         @log = log
         @read_length = read_length
-        @unpacker = Fluent::Engine.msgpack_unpacker
+        @unpacker = Fluent::MessagePackFactory.msgpack_unpacker
       end
 
       def collect_response(select_interval)

data/lib/fluent/plugin/out_forward/load_balancer.rb

@@ -36,8 +36,8 @@ module Fluent::Plugin
         wlen = @weight_array.size
         wlen.times do
           node = @mutex.synchronize do
-            r = @rr
-            @rr = (@rr + 1) % @weight_array.size
+            r = @rr % @weight_array.size
+            @rr = (r + 1) % @weight_array.size
             @weight_array[r]
           end
           next unless node.available?
@@ -106,6 +106,9 @@ module Fluent::Plugin
           @weight_array = weight_array
         end
       end
+
+      alias select_service select_healthy_node
+      alias rebalance rebuild_weight_array
     end
   end
 end

data/lib/fluent/plugin/out_stream.rb

@@ -68,7 +68,7 @@ module Fluent
         chain = NullOutputChain.instance
         chunk.open {|io|
           # TODO use MessagePackIoEventStream
-          u = Fluent::Engine.msgpack_factory.unpacker(io)
+          u = Fluent::MessagePackFactory.msgpack_unpacker(io)
           begin
             u.each {|(tag,entries)|
               es = MultiEventStream.new

data/lib/fluent/plugin/output.rb

@@ -40,7 +40,7 @@ module Fluent
       helpers_internal :thread, :retry_state
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
-      CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@$a-zA-Z0-9]+\}/
+      CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
       CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[-?\d+\])?)\}/
       CHUNK_ID_PLACEHOLDER_PATTERN = /\$\{chunk_id\}/
 
@@ -707,7 +707,7 @@ module Fluent
       end
 
       def get_placeholders_keys(str)
-        str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map{|ph| ph[2..-2]}.reject{|s| (s == "tag") || (s == 'chunk_id') }.sort
+        str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map(&:first).reject{|s| (s == "tag") || (s == 'chunk_id') }.sort
       end
 
       # TODO: optimize this code
@@ -759,11 +759,19 @@ module Fluent
             @chunk_keys.each do |key|
               hash["${#{key}}"] = metadata.variables[key.to_sym]
             end
-            rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN, hash)
+
+            rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN) do |matched|
+              hash.fetch(matched) do
+                log.warn "chunk key placeholder '#{matched[2..-2]}' not replaced. template:#{str}"
+                ''
+              end
+            end
           end
+
           if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
             log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
           end
+
           rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
             if chunk_passed
               dump_unique_id_hex(chunk.unique_id)

data/lib/fluent/plugin/parser.rb

@@ -35,6 +35,7 @@ module Fluent
           @flag = ServerEngine::BlockingFlag.new
           @mutex = Mutex.new
           @timeout = timeout
+          @timeout_checker = nil
         end
 
         def start

data/lib/fluent/plugin/sd_file.rb

@@ -0,0 +1,155 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+
+require 'fluent/plugin_helper'
+require 'fluent/plugin/service_discovery'
+
+module Fluent
+  module Plugin
+    class FileServiceDiscovery < ServiceDiscovery
+      include PluginHelper::Mixin
+
+      Plugin.register_sd('file', self)
+
+      DEFAULT_FILE_TYPE = :yaml
+      DEFAUT_WEIGHT = 60
+      DEFAULT_SD_FILE_PATH = ENV['DEFAULT_SD_FILE_PATH'] || '/etc/fluent/sd.yaml'
+
+      helpers :event_loop
+
+      config_param :path, :string, default: DEFAULT_SD_FILE_PATH
+      config_param :conf_encoding, :string, default: 'utf-8'
+
+      def initialize
+        super
+
+        @file_type = nil
+      end
+
+      def configure(conf)
+        super
+
+        unless File.exist?(@path)
+          raise Fluent::ConfigError, "sd_file: path=#{@path} not found"
+        end
+
+        @file_type = File.basename(@path).split('.', 2).last.to_sym
+        unless %i[yaml yml json].include?(@file_type)
+          @file_type = DEFAULT_FILE_TYPE
+        end
+
+        @services = fetch_server_info
+      end
+
+      def start(queue)
+        watcher = StatWatcher.new(@path, @log) do |_prev, _cur|
+          refresh_file(queue)
+        end
+        event_loop_attach(watcher)
+
+        super()
+      end
+
+      private
+
+      def parser
+        @parser ||=
+          case @file_type
+          when :yaml, :yml
+            require 'yaml'
+            -> (v) { YAML.safe_load(v).map }
+          when :json
+            require 'json'
+            -> (v) { JSON.parse(v) }
+          end
+      end
+
+      def refresh_file(queue)
+        s =
+          begin
+            fetch_server_info
+          rescue => e
+            @log.error("sd_file: #{e}")
+          end
+
+        if s.nil?
+          # if any error occurs, skip this turn
+          return
+        end
+
+        diff = []
+        join = s - @services
+        # Need service_in first to guarantee that server exist at least one all time.
+        join.each do |j|
+          diff << ServiceDiscovery.service_in_msg(j)
+        end
+
+        drain = @services - s
+        drain.each do |d|
+          diff << ServiceDiscovery.service_out_msg(d)
+        end
+
+        @services = s
+
+        diff.each do |a|
+          queue.push(a)
+        end
+      end
+
+      def fetch_server_info
+        config_data =
+          begin
+            File.open(@path, "r:#{@conf_encoding}:utf-8", &:read)
+          rescue => e
+            raise Fluent::ConfigError, "sd_file: path=#{@path} couldn't open #{e}"
+          end
+
+        parser.call(config_data).map do |s|
+          Service.new(
+            :file,
+            s.fetch('host'),
+            s.fetch('port'),
+            s['name'],
+            s.fetch('weight', DEFAUT_WEIGHT),
+            s['standby'],
+            s['username'],
+            s['password'],
+            s['shared_key'],
+          )
+        end
+      rescue KeyError => e
+        raise Fluent::ConfigError, "#{e}. Service must have `host` and `port`"
+      end
+
+      class StatWatcher < Coolio::StatWatcher
+        def initialize(path, log, &callback)
+          @path = path
+          @log = log
+          @callback = callback
+          super(@path)
+        end
+
+        def on_change(prev_stat, cur_stat)
+          @callback.call(prev_stat, cur_stat)
+        rescue => e
+          @log.error(e)
+        end
+      end
+    end
+  end
+end