fluentd 1.7.4 → 1.8.0.rc1

data/lib/fluent/plugin/sd_static.rb

@@ -0,0 +1,58 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/service_discovery'
+
+module Fluent
+  module Plugin
+    class StaticServiceDiscovery < ServiceDiscovery
+      Plugin.register_sd('static', self)
+
+      LISTEN_PORT = 24224
+
+      config_section :service, param_name: :service_configs do
+        desc 'The IP address or host name of the server.'
+        config_param :host, :string
+        desc 'The name of the server. Used for logging and certificate verification in TLS transport (when host is address).'
+        config_param :name, :string, default: nil
+        desc 'The port number of the host.'
+        config_param :port, :integer, default: LISTEN_PORT
+        desc 'The shared key per server.'
+        config_param :shared_key, :string, default: nil, secret: true
+        desc 'The username for authentication.'
+        config_param :username, :string, default: ''
+        desc 'The password for authentication.'
+        config_param :password, :string, default: '', secret: true
+        desc 'Marks a node as the standby node for an Active-Standby model between Fluentd nodes.'
+        config_param :standby, :bool, default: false
+        desc 'The load balancing weight.'
+        config_param :weight, :integer, default: 60
+      end
+
+      def configure(conf)
+        super
+
+        @services = @service_configs.map do |s|
+          ServiceDiscovery::Service.new(:static, s.host, s.port, s.name, s.weight, s.standby, s.username, s.password, s.shared_key)
+        end
+      end
+
+      def start(queue = nil)
+        super()
+      end
+    end
+  end
+end

data/lib/fluent/plugin/service_discovery.rb

@@ -0,0 +1,80 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/unique_id'
+require 'fluent/plugin_id'
+
+module Fluent
+  module Plugin
+    class ServiceDiscovery < Base
+      include PluginId
+      include PluginLoggerMixin
+      include UniqueId::Mixin
+
+      configured_in :service_discovery
+
+      attr_reader :services
+
+      Service = Struct.new(:plugin_name, :host, :port, :name, :weight, :standby, :username, :password, :shared_key) do
+        def discovery_id
+          @discovery_id ||= Base64.encode64(to_h.to_s)
+        end
+      end
+
+      SERVICE_IN = :service_in
+      SERVICE_OUT = :service_out
+      DiscoveryMessage = Struct.new(:type, :service)
+
+      class << self
+        def service_in_msg(service)
+          DiscoveryMessage.new(SERVICE_IN, service)
+        end
+
+        def service_out_msg(service)
+          DiscoveryMessage.new(SERVICE_OUT, service)
+        end
+      end
+
+      def initialize
+        @services = []
+
+        super
+      end
+
+      def start(queue = nil)
+        super()
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper.rb

@@ -31,6 +31,7 @@ require 'fluent/plugin_helper/counter'
 require 'fluent/plugin_helper/retry_state'
 require 'fluent/plugin_helper/record_accessor'
 require 'fluent/plugin_helper/compat_parameters'
+require 'fluent/plugin_helper/service_discovery'
 
 module Fluent
   module PluginHelper

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -31,6 +31,10 @@ module Fluent
         Accessor.new(param)
       end
 
+      def record_accessor_nested?(param)
+        Accessor.parse_parameter(param).is_a?(Array)
+      end
+
       class Accessor
         attr_reader :keys
 
@@ -42,25 +46,19 @@ module Fluent
             @dig_keys = @keys[0..-2]
             if @dig_keys.empty?
               @keys = @keys.first
-              mcall = method(:call_index)
-              mdelete = method(:delete_top)
             else
               mcall = method(:call_dig)
               mdelete = method(:delete_nest)
+              singleton_class.module_eval do
+                define_method(:call, mcall)
+                define_method(:delete, mdelete)
+              end
             end
-          else
-            # Call [] for single key to reduce dig overhead
-            mcall = method(:call_index)
-            mdelete = method(:delete_top)
-          end
-
-          singleton_class.module_eval do
-            define_method(:call, mcall)
-            define_method(:delete, mdelete)
           end
         end
 
         def call(r)
+          r[@keys]
         end
 
         # To optimize the performance, use class_eval with pre-expanding @keys
@@ -69,11 +67,8 @@ module Fluent
           r.dig(*@keys)
         end
 
-        def call_index(r)
-          r[@keys]
-        end
-
         def delete(r)
+          r.delete(@keys)
         end
 
         def delete_nest(r)
@@ -88,10 +83,6 @@ module Fluent
           nil
         end
 
-        def delete_top(r)
-          r.delete(@keys)
-        end
-
         def self.parse_parameter(param)
           if param.start_with?('$.')
             parse_dot_notation(param)

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -0,0 +1,80 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/timer'
+require 'fluent/plugin_helper/service_discovery/manager'
+
+module Fluent
+  module PluginHelper
+    module ServiceDiscovery
+      include Fluent::PluginHelper::Timer
+
+      def self.included(mod)
+        mod.include ServiceDiscoveryParams
+      end
+
+      def start
+        unless @discovery_manager
+          log.warn('There is no discovery_manager. skip start them')
+          super
+          return
+        end
+
+        @discovery_manager.start
+        unless @discovery_manager.static_config?
+          timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_iterval) do
+            @discovery_manager.run_once
+          end
+        end
+
+        super
+      end
+
+      private
+
+      # @param title [Symbol] the thread name. this value should be unique.
+      # @param configurations [Hash] hash which must has discivery_service type and its configuration like `{ type: :static, conf: <Fluent::Config::Element> }`
+      # @param load_balancer [Object] object which has two methods #rebalance and #select_service
+      # @param custom_build_method [Proc]
+      def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
+        @_plugin_helper_service_discovery_title = title
+        @_plugin_helper_service_discovery_iterval = interval
+
+        @discovery_manager = Fluent::PluginHelper::ServiceDiscovery::Manager.new(
+          log: log,
+          load_balancer: load_balancer,
+          custom_build_method: custom_build_method,
+        )
+
+        @discovery_manager.configure(configurations, parent: self)
+
+        @discovery_manager
+      end
+
+      def discovery_manager
+        @discovery_manager
+      end
+
+      module ServiceDiscoveryParams
+        include Fluent::Configurable
+
+        config_section :service_discovery do
+          config_param :@type, :string
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/service_discovery/manager.rb

@@ -0,0 +1,132 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/service_discovery'
+require 'fluent/plugin_helper/service_discovery/round_robin_balancer'
+
+module Fluent
+  module PluginHelper
+    module ServiceDiscovery
+      class Manager
+        def initialize(log:, load_balancer: nil, custom_build_method: nil)
+          @log = log
+          @load_balancer = load_balancer || RoundRobinBalancer.new
+          @custom_build_method = custom_build_method
+
+          @discoveries = []
+          @services = {}
+          @queue = Queue.new
+          @static_config = true
+        end
+
+        def configure(opts, parent: nil)
+          opts.each do |opt|
+            sd = Fluent::Plugin.new_sd(opt[:type], parent: parent)
+            sd.configure(opt[:conf])
+
+            sd.services.each do |s|
+              @services[s.discovery_id] = build_service(s)
+            end
+            @discoveries << sd
+
+            if @static_config && opt[:type] != :static
+              @static_config = false
+            end
+          end
+
+          rebalance
+        end
+
+        def static_config?
+          @static_config
+        end
+
+        def start
+          @discoveries.each do |d|
+            d.start(@queue)
+          end
+        end
+
+        def run_once
+          # Don't care race in this loop intentionally
+          s = @queue.size
+
+          if s == 0
+            return
+          end
+
+          s.times do
+            msg = @queue.pop
+
+            unless msg.is_a?(Fluent::Plugin::ServiceDiscovery::DiscoveryMessage)
+              @log.warn("BUG: #{msg}")
+              next
+            end
+
+            begin
+              handle_message(msg)
+            rescue => e
+              @log.error(e)
+            end
+          end
+
+          rebalance
+        end
+
+        def rebalance
+          @load_balancer.rebalance(services)
+        end
+
+        def select_service(&block)
+          @load_balancer.select_service(&block)
+        end
+
+        def services
+          @services.values
+        end
+
+        private
+
+        def handle_message(msg)
+          service = msg.service
+
+          case msg.type
+          when Fluent::Plugin::ServiceDiscovery::SERVICE_IN
+            if (n = build_service(service))
+              @log.info("Service in: name=#{service.name} #{service.host}:#{service.port}")
+              @services[service.discovery_id] = n
+            else
+              raise "failed to build service in name=#{service.name} #{service.host}:#{service.port}"
+            end
+          when Fluent::Plugin::ServiceDiscovery::SERVICE_OUT
+            s = @services.delete(service.discovery_id)
+            if s
+              @log.info("Service out: name=#{service.name} #{service.host}:#{service.port}")
+            else
+              @log.warn("Not found service: name=#{service.name} #{service.host}:#{service.port}")
+            end
+          else
+            @log.error("BUG: unknow message type: #{msg.type}")
+          end
+        end
+
+        def build_service(n)
+          @custom_build_method ? @custom_build_method.call(n) : n
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb

@@ -0,0 +1,43 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module ServiceDiscovery
+      class RoundRobinBalancer
+        def initialize
+          @services = []
+          @mutex = Mutex.new
+        end
+
+        def rebalance(services)
+          @mutex.synchronize do
+            @services = services
+          end
+        end
+
+        def select_service
+          s = @mutex.synchronize do
+            s = @services.shift
+            @services.push(s)
+            s
+          end
+          yield(s)
+        end
+      end
+    end
+  end
+end