fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin_helper/counter.rb

@@ -0,0 +1,51 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/counter/client'
+
+module Fluent
+  module PluginHelper
+    module Counter
+      def counter_client_create(scope:, loop: Coolio::Loop.new)
+        client_conf = system_config.counter_client
+        raise Fluent::ConfigError, '<counter_client> is required in <system>' unless client_conf
+        counter_client = Fluent::Counter::Client.new(loop, port: client_conf.port, host: client_conf.host, log: log, timeout: client_conf.timeout)
+        counter_client.start
+        counter_client.establish(scope)
+        @_counter_client = counter_client
+        counter_client
+      end
+
+      attr_reader :_counter_client
+
+      def initialize
+        super
+        @_counter_client = nil
+      end
+
+      def stop
+        super
+        @_counter_client.stop
+      end
+
+      def terminate
+        @_counter_client = nil
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/event_emitter.rb

@@ -0,0 +1,93 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/time'
+
+module Fluent
+  module PluginHelper
+    module EventEmitter
+      # stop     : [-]
+      # shutdown : disable @router
+      # close    : [-]
+      # terminate: [-]
+
+      def router
+        @_event_emitter_used_actually = true
+        if @_event_emitter_lazy_init
+          @router = @primary_instance.router
+        end
+        @router
+      end
+
+      def router=(r)
+        # not recommended now...
+        @router = r
+      end
+
+      def has_router?
+        true
+      end
+
+      def event_emitter_used_actually?
+        @_event_emitter_used_actually
+      end
+
+      def event_emitter_router(label_name)
+        if label_name
+          Engine.root_agent.find_label(label_name).event_router
+        elsif self.respond_to?(:as_secondary) && self.as_secondary
+          if @primary_instance.has_router?
+            @_event_emitter_lazy_init = true
+            nil # primary plugin's event router is not initialized yet, here.
+          else
+            @primary_instance.context_router
+          end
+        else
+          # `Engine.root_agent.event_router` is for testing
+          self.context_router || Engine.root_agent.event_router
+        end
+      end
+
+      def initialize
+        super
+        @_event_emitter_used_actually = false
+        @_event_emitter_lazy_init = false
+        @router = nil
+      end
+
+      def configure(conf)
+        require 'fluent/engine'
+        super
+        @router = event_emitter_router(conf['@label'])
+      end
+
+      def after_shutdown
+        @router = nil
+        super
+      end
+
+      def close # unset router many times to reduce test cost
+        @router = nil
+        super
+      end
+
+      def terminate
+        @router = nil
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/event_loop.rb

@@ -0,0 +1,170 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'fluent/plugin_helper/thread'
+require 'fluent/clock'
+
+module Fluent
+  module PluginHelper
+    module EventLoop
+      # Currently this plugin helper is only for other helpers, not plugins.
+      # there's no way to create customized watchers to attach event loops.
+      include Fluent::PluginHelper::Thread
+
+      # stop     : [-]
+      # shutdown : detach all event watchers on event loop
+      # close    : stop event loop
+      # terminate: initialize internal state
+
+      EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
+      EVENT_LOOP_SHUTDOWN_TIMEOUT = 5
+
+      attr_reader :_event_loop # for tests
+
+      def event_loop_attach(watcher)
+        @_event_loop_mutex.synchronize do
+          @_event_loop.attach(watcher)
+          @_event_loop_attached_watchers << watcher
+          watcher
+        end
+      end
+
+      def event_loop_detach(watcher)
+        if watcher.attached?
+          watcher.detach
+        end
+        @_event_loop_mutex.synchronize do
+          @_event_loop_attached_watchers.delete(watcher)
+        end
+      end
+
+      def event_loop_wait_until_start
+        sleep(0.1) until event_loop_running?
+      end
+
+      def event_loop_wait_until_stop
+        timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
+        sleep(0.1) while event_loop_running? && Fluent::Clock.now < timeout_at
+        if @_event_loop_running
+          puts "terminating event_loop forcedly"
+          caller.each{|bt| puts "\t#{bt}" }
+          @_event_loop.stop rescue nil
+          @_event_loop_running = true
+        end
+      end
+
+      def event_loop_running?
+        @_event_loop_running
+      end
+
+      def initialize
+        super
+        @_event_loop = Coolio::Loop.new
+        @_event_loop_running = false
+        @_event_loop_mutex = Mutex.new
+        # plugin MAY configure loop run timeout in #configure
+        @_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT
+        @_event_loop_attached_watchers = []
+      end
+
+      def start
+        super
+
+        # event loop does not run here, so mutex lock is not required
+        thread_create :event_loop do
+          begin
+            default_watcher = DefaultWatcher.new
+            event_loop_attach(default_watcher)
+            @_event_loop_running = true
+            @_event_loop.run(@_event_loop_run_timeout) # this method blocks
+          ensure
+            @_event_loop_running = false
+          end
+        end
+      end
+
+      def shutdown
+        @_event_loop_mutex.synchronize do
+          @_event_loop_attached_watchers.reverse.each do |w|
+            if w.attached?
+              begin
+                w.detach
+              rescue => e
+                log.warn "unexpected error while detaching event loop watcher", error: e
+              end
+            end
+          end
+        end
+
+        super
+      end
+
+      def after_shutdown
+        timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
+        @_event_loop_mutex.synchronize do
+          @_event_loop.watchers.reverse.each do |w|
+            begin
+              w.detach
+            rescue => e
+              log.warn "unexpected error while detaching event loop watcher", error: e
+            end
+          end
+        end
+        while @_event_loop_running
+          if Fluent::Clock.now >= timeout_at
+            log.warn "event loop does NOT exit until hard timeout."
+            raise "event loop does NOT exit until hard timeout." if @under_plugin_development
+            break
+          end
+          sleep 0.1
+        end
+
+        super
+      end
+
+      def close
+        if @_event_loop_running
+          begin
+            @_event_loop.stop # we cannot check loop is running or not
+          rescue RuntimeError => e
+            raise unless e.message == 'loop not running'
+          end
+        end
+
+        super
+      end
+
+      def terminate
+        @_event_loop = nil
+        @_event_loop_running = false
+        @_event_loop_mutex = nil
+        @_event_loop_run_timeout = nil
+
+        super
+      end
+
+      # watcher to block to run event loop until shutdown
+      class DefaultWatcher < Coolio::TimerWatcher
+        def initialize
+          super(1, true) # interval: 1, repeat: true
+        end
+        # do nothing
+        def on_timer; end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/extract.rb

@@ -0,0 +1,104 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+require 'fluent/time'
+require 'fluent/configurable'
+
+module Fluent
+  module PluginHelper
+    module Extract
+      def extract_tag_from_record(record)
+        return nil unless @_extract_enabled
+
+        if @_extract_tag_key && record.has_key?(@_extract_tag_key)
+          v = @_extract_keep_tag_key ? record[@_extract_tag_key] : record.delete(@_extract_tag_key)
+          return v.to_s
+        end
+
+        nil
+      end
+
+      def extract_time_from_record(record)
+        return nil unless @_extract_enabled
+
+        if @_extract_time_key && record.has_key?(@_extract_time_key)
+          v = @_extract_keep_time_key ? record[@_extract_time_key] : record.delete(@_extract_time_key)
+          return @_extract_time_parser.call(v)
+        end
+
+        nil
+      end
+
+      module ExtractParams
+        include Fluent::Configurable
+        config_section :extract, required: false, multi: false, param_name: :extract_config do
+          config_param :tag_key, :string, default: nil
+          config_param :keep_tag_key, :bool, default: false
+          config_param :time_key, :string, default: nil
+          config_param :keep_time_key, :bool, default: false
+
+          # To avoid defining :time_type twice
+          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+
+          Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
+            config_param name, type, opts
+          end
+        end
+      end
+
+      def self.included(mod)
+        mod.include ExtractParams
+      end
+
+      def initialize
+        super
+        @_extract_enabled = false
+        @_extract_tag_key = nil
+        @_extract_keep_tag_key = nil
+        @_extract_time_key = nil
+        @_extract_keep_time_key = nil
+        @_extract_time_parser = nil
+      end
+
+      def configure(conf)
+        super
+
+        if @extract_config
+          @_extract_tag_key = @extract_config.tag_key
+          @_extract_keep_tag_key = @extract_config.keep_tag_key
+          @_extract_time_key = @extract_config.time_key
+          if @_extract_time_key
+            @_extract_keep_time_key = @extract_config.keep_time_key
+            @_extract_time_parser = case @extract_config.time_type
+                                    when :float then Fluent::NumericTimeParser.new(:float)
+                                    when :unixtime then Fluent::NumericTimeParser.new(:unixtime)
+                                    else
+                                      localtime = @extract_config.localtime && !@extract_config.utc
+                                      Fluent::TimeParser.new(@extract_config.time_format, localtime, @extract_config.timezone)
+                                    end
+          else
+            if @extract_config.time_format
+              log.warn "'time_format' specified without 'time_key', will be ignored"
+            end
+          end
+
+          @_extract_enabled = @_extract_tag_key || @_extract_time_key
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/formatter.rb

@@ -0,0 +1,147 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/formatter'
+require 'fluent/config/element'
+require 'fluent/configurable'
+
+module Fluent
+  module PluginHelper
+    module Formatter
+      def formatter_create(usage: '', type: nil, conf: nil, default_type: nil)
+        formatter = @_formatters[usage]
+        return formatter if formatter && !type && !conf
+
+        type = if type
+                 type
+               elsif conf && conf.respond_to?(:[])
+                 raise Fluent::ConfigError, "@type is required in <format>" unless conf['@type']
+                 conf['@type']
+               elsif default_type
+                 default_type
+               else
+                 raise ArgumentError, "BUG: both type and conf are not specified"
+               end
+        formatter = Fluent::Plugin.new_formatter(type, parent: self)
+        config = case conf
+                 when Fluent::Config::Element
+                   conf
+                 when Hash
+                   # in code, programmer may use symbols as keys, but Element needs strings
+                   conf = Hash[conf.map{|k,v| [k.to_s, v]}]
+                   Fluent::Config::Element.new('format', usage, conf, [])
+                 when nil
+                   Fluent::Config::Element.new('format', usage, {}, [])
+                 else
+                   raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
+                 end
+        formatter.configure(config)
+        if @_formatters_started
+          formatter.start
+        end
+
+        @_formatters[usage] = formatter
+        formatter
+      end
+
+      module FormatterParams
+        include Fluent::Configurable
+        # minimum section definition to instantiate formatter plugin instances
+        config_section :format, required: false, multi: true, init: true, param_name: :formatter_configs do
+          config_argument :usage, :string, default: ''
+          config_param    :@type, :string # config_set_default required for :@type
+        end
+      end
+
+      def self.included(mod)
+        mod.include FormatterParams
+      end
+
+      attr_reader :_formatters # for tests
+
+      def initialize
+        super
+        @_formatters_started = false
+        @_formatters = {} # usage => formatter
+      end
+
+      def configure(conf)
+        super
+
+        @formatter_configs.each do |section|
+          if @_formatters[section.usage]
+            raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
+          end
+          formatter = Plugin.new_formatter(section[:@type], parent: self)
+          formatter.configure(section.corresponding_config_element)
+          @_formatters[section.usage] = formatter
+        end
+      end
+
+      def start
+        super
+        @_formatters_started = true
+        @_formatters.each_pair do |usage, formatter|
+          formatter.start
+        end
+      end
+
+      def formatter_operate(method_name, &block)
+        @_formatters.each_pair do |usage, formatter|
+          begin
+            formatter.send(method_name)
+            block.call(formatter) if block_given?
+          rescue => e
+            log.error "unexpected error while #{method_name}", usage: usage, formatter: formatter, error: e
+          end
+        end
+      end
+
+      def stop
+        super
+        formatter_operate(:stop)
+      end
+
+      def before_shutdown
+        formatter_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        formatter_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        formatter_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        formatter_operate(:close)
+        super
+      end
+
+      def terminate
+        formatter_operate(:terminate)
+        @_formatters_started = false
+        @_formatters = {}
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/app.rb

@@ -0,0 +1,79 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async/http/protocol'
+require 'fluent/plugin_helper/http_server/methods'
+require 'fluent/plugin_helper/http_server/request'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class App
+        def initialize(router, logger)
+          @logger = logger
+          @router = router
+        end
+
+        # Required method by async-http
+        def call(request)
+          method = request.method
+          resp =
+            case method
+            when HttpServer::Methods::GET
+              get(request)
+            when HttpServer::Methods::HEAD
+              head(request)
+            when HttpServer::Methods::POST
+              post(request)
+            when HttpServer::Methods::PATCH
+              patch(request)
+            when HttpServer::Methods::PUT
+              put(request)
+            when HttpServer::Methods::DELETE
+              delete(request)
+            when HttpServer::Methods::OPTIONS
+              options(request)
+            when HttpServer::Methods::CONNECT
+              connect(request)
+            when HttpServer::Methods::TRACE
+              trace(request)
+            else
+              raise "Unknown method #{method}"
+            end
+          Protocol::HTTP::Response[*resp]
+        rescue => e
+          @logger.error(e)
+          Protocol::HTTP::Response[500, { 'Content-Type' => 'text/plain' }, 'Internal Server Error']
+        end
+
+        HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
+          define_method(name) do |request|
+            req = Request.new(request)
+
+            path = req.path
+            canonical_path =
+              if path.size >= 2 && !path.end_with?('/')
+                "#{path}/"
+              else
+                path
+              end
+            @router.route!(name, canonical_path, req)
+          end
+        end
+      end
+    end
+  end
+end