fluentd 0.12.40 → 0.14.0

data/lib/fluent/agent.rb

@@ -15,7 +15,6 @@
 #
 
 require 'fluent/configurable'
-require 'fluent/engine'
 require 'fluent/plugin'
 require 'fluent/output'
 
@@ -29,16 +28,23 @@ module Fluent
   class Agent
     include Configurable
 
-    def initialize(opts = {})
+    def initialize(log:)
       super()
 
       @context = nil
       @outputs = []
       @filters = []
-      @started_outputs = []
-      @started_filters = []
 
-      @log = Engine.log
+      @lifecycle_control_list = nil
+      # lifecycle_control_list is the list of plugins in this agent, and ordered
+      # from plugins which DOES emit, then DOESN'T emit
+      # (input -> output w/ router -> filter -> output w/o router)
+      # for start: use this order DESC
+      #   (because plugins which appears later in configurations will receive events from plugins which appears ealier)
+      # for stop/before_shutdown/shutdown/after_shutdown/close/terminate: use this order ASC
+      @lifecycle_cache = nil
+
+      @log = log
       @event_router = EventRouter.new(NoMatchMatch.new(log), self)
       @error_collector = nil
     end
@@ -54,10 +60,9 @@ module Fluent
       super
 
       # initialize <match> and <filter> elements
-      conf.elements.select { |e| e.name == 'filter' || e.name == 'match' }.each { |e|
+      conf.elements('filter', 'match').each { |e|
         pattern = e.arg.empty? ? '**' : e.arg
-        type = e['@type'] || e['type']
-        raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type
+        type = e['@type']
         if e.name == 'filter'
           add_filter(type, pattern, e)
         else
@@ -66,70 +71,67 @@ module Fluent
       }
     end
 
-    def start
-      @outputs.each { |o|
-        o.start
-        @started_outputs << o
-      }
+    def lifecycle_control_list
+      return @lifecycle_control_list if @lifecycle_control_list
 
-      @filters.each { |f|
-        f.start
-        @started_filters << f
+      lifecycle_control_list = {
+        input: [],
+        output_with_router: [],
+        filter: [],
+        output: [],
       }
-    end
-
-    def shutdown
-      @started_filters.map { |f|
-        Thread.new do
-          begin
-            log.info "shutting down filter#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(f.class), plugin_id: f.plugin_id
-            f.shutdown
-          rescue => e
-            log.warn "unexpected error while shutting down filter plugins", plugin: f.class, plugin_id: f.plugin_id, error_class: e.class, error: e
-            log.warn_backtrace
-          end
+      if self.respond_to?(:inputs)
+        inputs.each do |i|
+          lifecycle_control_list[:input] << i
+        end
+      end
+      recursive_output_traverse = ->(o) {
+        if o.has_router?
+          lifecycle_control_list[:output_with_router] << o
+        else
+          lifecycle_control_list[:output] << o
         end
-      }.each { |t| t.join }
-
-      # Output plugin as filter emits records at shutdown so emit problem still exist.
-      # This problem will be resolved after actual filter mechanizm.
-      @started_outputs.map { |o|
-        Thread.new do
-          begin
-            log.info "shutting down output#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(o.class), plugin_id: o.plugin_id
-            o.shutdown
-          rescue => e
-            log.warn "unexpected error while shutting down output plugins", plugin: o.class, plugin_id: o.plugin_id, error_class: e.class, error: e
-            log.warn_backtrace
+
+        if o.respond_to?(:outputs)
+          o.outputs.each do |store|
+            recursive_output_traverse.call(store)
           end
         end
-      }.each { |t| t.join }
-    end
+      }
+      outputs.each do |o|
+        recursive_output_traverse.call(o)
+      end
+      filters.each do |f|
+        lifecycle_control_list[:filter] << f
+      end
 
-    def flush!
-      flush_recursive(@outputs)
+      @lifecycle_control_list = lifecycle_control_list
     end
 
-    def flush_recursive(array)
-      array.each { |o|
-        begin
-          if o.is_a?(BufferedOutput)
-            o.force_flush
-          elsif o.is_a?(MultiOutput)
-            flush_recursive(o.outputs)
-          end
-        rescue => e
-          log.debug "error while force flushing", error_class: e.class, error: e
-          log.debug_backtrace
+    def lifecycle(desc: false)
+      kind_list = if desc
+                    [:output, :filter, :output_with_router]
+                  else
+                    [:output_with_router, :filter, :output]
+                  end
+      kind_list.each do |kind|
+        list = if desc
+                 lifecycle_control_list[kind].reverse
+               else
+                 lifecycle_control_list[kind]
+               end
+        display_kind = (kind == :output_with_router ? :output : kind)
+        list.each do |instance|
+          yield instance, display_kind
         end
-      }
+      end
     end
 
     def add_match(type, pattern, conf)
       log.info "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
 
       output = Plugin.new_output(type)
-      output.router = @event_router
+      output.router = @event_router if output.respond_to?(:router=)
       output.configure(conf)
       @outputs << output
       @event_router.add_rule(pattern, output)
@@ -162,7 +164,7 @@ module Fluent
         @count = 0
       end
 
-      def emit(tag, es, chain)
+      def emit_events(tag, es)
         # TODO use time instead of num of records
         c = (@count += 1)
         if c < 512

data/lib/fluent/command/cat.rb

@@ -21,7 +21,7 @@ op = OptionParser.new
 
 op.banner += " <tag>"
 
-port = Fluent::DEFAULT_LISTEN_PORT
+port = 24224
 host = '127.0.0.1'
 unix = false
 socket_path = Fluent::DEFAULT_SOCKET_PATH

data/lib/fluent/command/debug.rb

@@ -62,13 +62,15 @@ end
 
 require 'fluent/log'
 require 'fluent/engine'
-require 'fluent/config'
 require 'fluent/system_config'
 
-$log = Fluent::Log.new(STDERR, Fluent::Log::LEVEL_TRACE)
-conf = Fluent::Config.parse('', 'fluent-debug.conf', '', true)
-system_conf = Fluent::SystemConfig.create(conf)
-Fluent::Engine.init(system_conf)
+include Fluent::SystemConfig::Mixin
+
+dl_opts = {}
+dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE
+logger = ServerEngine::DaemonLogger.new(STDERR, dl_opts)
+$log = Fluent::Log.new(logger)
+Fluent::Engine.init(system_config)
 
 DRb::DRbObject.class_eval do
   undef_method :methods

data/lib/fluent/command/fluentd.rb

@@ -17,8 +17,12 @@
 require 'optparse'
 
 require 'fluent/supervisor'
+require 'fluent/log'
+require 'fluent/env'
 require 'fluent/version'
 
+$fluentdargv = Marshal.load(Marshal.dump(ARGV))
+
 op = OptionParser.new
 op.version = Fluent::VERSION
 
@@ -72,7 +76,7 @@ op.on('-o', '--log PATH', "log file path") {|s|
   opts[:log_path] = s
 }
 
-op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-fly") {|s|
+op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-the-fly") {|s|
   opts[:inline_config] = s
 }
 
@@ -121,6 +125,28 @@ op.on('-G', '--gem-path GEM_INSTALL_PATH', "Gemfile install path (default: $(dir
   opts[:gem_install_path] = s
 }
 
+if Fluent.windows?
+  require 'windows/library'
+  include Windows::Library
+
+  op.on('-x', '--signame INTSIGNAME', "an object name which is used for Windows Service signal (Windows only)") {|s|
+    opts[:signame] = s
+  }
+
+  op.on('--reg-winsvc MODE', "install/uninstall as Windows Service. (i: install, u: uninstall) (Windows only)") {|s|
+    opts[:regwinsvc] = s
+  }
+
+  op.on('--[no-]reg-winsvc-auto-start', "Automatically start the Windows Service at boot. (only effective with '--reg-winsvc i') (Windows only)") {|s|
+    opts[:regwinsvcautostart] = s
+  }
+
+  op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option paramters for Windows Service. (Windows only)") {|s|
+    opts[:fluentdopt] = s
+  }
+end
+
+
 (class << self; self; end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
@@ -170,4 +196,73 @@ if setup_path = opts[:setup_path]
   exit 0
 end
 
-Fluent::Supervisor.new(opts).start
+early_exit = false
+start_service = false
+if winsvcinstmode = opts[:regwinsvc]
+  FLUENTD_WINSVC_NAME="fluentdwinsvc"
+  FLUENTD_WINSVC_DISPLAYNAME="Fluentd Windows Service"
+  FLUENTD_WINSVC_DESC="Fluentd is an event collector system."
+  require 'fileutils'
+  require "win32/service"
+  require "win32/registry"
+  include Win32
+
+  case winsvcinstmode
+  when 'i'
+    binary_path = File.join(File.dirname(__FILE__), "..")
+    ruby_path = "\0" * 256
+    GetModuleFileName.call(0,ruby_path,256)
+    ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
+    start_type = Service::DEMAND_START
+    if opts[:regwinsvcautostart]
+      start_type = Service::AUTO_START
+      start_service = true
+    end
+
+    Service.create(
+      service_name: FLUENTD_WINSVC_NAME,
+      host: nil,
+      service_type: Service::WIN32_OWN_PROCESS,
+      description: FLUENTD_WINSVC_DESC,
+      start_type: start_type,
+      error_control: Service::ERROR_NORMAL,
+      binary_path_name: ruby_path+" -C "+binary_path+" winsvc.rb",
+      load_order_group: "",
+      dependencies: [""],
+      display_name: FLUENTD_WINSVC_DISPLAYNAME
+    )
+  when 'u'
+    if Service.status(FLUENTD_WINSVC_NAME).current_state != 'stopped'
+      begin
+        Service.stop(FLUENTD_WINSVC_NAME)
+      rescue => ex
+        puts "Warning: Failed to stop service: ", ex
+      end
+    end
+    Service.delete(FLUENTD_WINSVC_NAME)
+  else
+    # none
+  end
+  early_exit = true
+end
+
+if fluentdopt = opts[:fluentdopt]
+  Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\fluentdwinsvc", Win32::Registry::KEY_ALL_ACCESS) do |reg|
+    reg['fluentdopt', Win32::Registry::REG_SZ] = fluentdopt
+  end
+  early_exit = true
+end
+
+if start_service
+  Service.start(FLUENTD_WINSVC_NAME)
+end
+
+exit 0 if early_exit
+
+require 'fluent/supervisor'
+if opts[:supervise]
+  Fluent::Supervisor.new(opts).run_supervisor
+else
+  Fluent::Supervisor.new(opts).run_worker
+end
+

data/lib/fluent/compat/call_super_mixin.rb

@@ -0,0 +1,67 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module CallSuperMixin
+      # This mixin is to prepend to 3rd party plugins of v0.12 APIs.
+      # In past, there were not strong rule to call super in #start, #before_shutdown and #shutdown.
+      # But v0.14 API requires to call super in these methods to setup/teardown plugin helpers and others.
+      # This mixin prepends method calls to call super forcedly if checker returns false (it shows Fluent::Plugin::Base#methods wasn't called)
+
+      def self.prepended(klass)
+        @@_super_start ||= {}
+        @@_super_before_shutdown ||= {}
+        @@_super_shutdown ||= {}
+
+        # ancestors[0]: this module
+        # ancestors[1]: prepended class (plugin itself)
+        method_search = ->(ancestors, method){
+          closest = ancestors[2, ancestors.size - 2].index{|m| m.method_defined?(method) }
+          ancestors[2 + closest].instance_method(method)
+        }
+        @@_super_start[klass]           = method_search.call(klass.ancestors, :start) # this returns Fluent::Compat::*#start (or helpers on it)
+        @@_super_before_shutdown[klass] = method_search.call(klass.ancestors, :before_shutdown)
+        @@_super_shutdown[klass]        = method_search.call(klass.ancestors, :shutdown)
+      end
+
+      def start
+        super
+        unless self.started?
+          @@_super_start[self.class].bind(self).call
+          # #super will reset logdev (especially in test), so this warn should be after calling it
+          log.warn "super was not called in #start: called it forcedly", plugin: self.class
+        end
+      end
+
+      def before_shutdown
+        super
+        unless self.before_shutdown?
+          log.warn "super was not called in #before_shutdown: calling it forcedly", plugin: self.class
+          @@_super_before_shutdown[self.class].bind(self).call
+        end
+      end
+
+      def shutdown
+        super
+        unless self.shutdown?
+          log.warn "super was not called in #shutdown: calling it forcedly", plugin: self.class
+          @@_super_shutdown[self.class].bind(self).call
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/filter.rb

@@ -0,0 +1,50 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/filter'
+require 'fluent/compat/call_super_mixin'
+
+module Fluent
+  module Compat
+    class Filter < Fluent::Plugin::Filter
+      # TODO: warn when deprecated
+
+      def initialize
+        super
+        unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
+          self.class.module_eval do
+            prepend Fluent::Compat::CallSuperMixin
+          end
+        end
+      end
+
+      # These definitions are to get instance methods of superclass of 3rd party plugins
+      # to make it sure to call super
+      def start
+        super
+      end
+
+      def before_shutdown
+        super
+      end
+
+      def shutdown
+        super
+      end
+    end
+  end
+end

data/lib/fluent/compat/formatter.rb

@@ -0,0 +1,109 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/formatter'
+
+require 'fluent/plugin/formatter_out_file'
+require 'fluent/plugin/formatter_stdout'
+require 'fluent/plugin/formatter_json'
+require 'fluent/plugin/formatter_hash'
+require 'fluent/plugin/formatter_msgpack'
+require 'fluent/plugin/formatter_ltsv'
+require 'fluent/plugin/formatter_csv'
+require 'fluent/plugin/formatter_single_value'
+
+module Fluent
+  module Compat
+    class Formatter < Fluent::Plugin::Formatter
+      # TODO: warn when deprecated
+    end
+
+    module TextFormatter
+      def self.register_template(type, template)
+        # TODO: warn when deprecated to use Plugin.register_formatter directly
+        if template.is_a?(Class)
+          Fluent::Plugin.register_formatter(type, template)
+        elsif template.respond_to?(:call) && template.arity == 3 # Proc.new { |tag, time, record| }
+          Fluent::Plugin.register_formatter(type, Proc.new { ProcWrappedFormatter.new(template) })
+        elsif template.respond_to?(:call)
+          Fluent::Plugin.register_formatter(type, template)
+        else
+          raise ArgumentError, "Template for formatter must be a Class or callable object"
+        end
+      end
+
+      def self.lookup(type)
+        # TODO: warn when deprecated to use Plugin.new_formatter(type, parent: plugin)
+        Fluent::Plugin.new_formatter(type)
+      end
+
+      # Keep backward-compatibility
+      def self.create(conf)
+        # TODO: warn when deprecated
+        format = conf['format']
+        if format.nil?
+          raise ConfigError, "'format' parameter is required"
+        end
+
+        formatter = lookup(format)
+        if formatter.respond_to?(:configure)
+          formatter.configure(conf)
+        end
+        formatter
+      end
+
+      HandleTagAndTimeMixin = Fluent::Plugin::Formatter::HandleTagAndTimeMixin
+      StructuredFormatMixin = Fluent::Plugin::Formatter::StructuredFormatMixin
+
+      class ProcWrappedFormatter < Fluent::Plugin::ProcWrappedFormatter
+        # TODO: warn when deprecated
+      end
+
+      class OutFileFormatter < Fluent::Plugin::OutFileFormatter
+        # TODO: warn when deprecated
+      end
+
+      class StdoutFormatter < Fluent::Plugin::StdoutFormatter
+        # TODO: warn when deprecated
+      end
+
+      class JSONFormatter < Fluent::Plugin::JSONFormatter
+        # TODO: warn when deprecated
+      end
+
+      class HashFormatter < Fluent::Plugin::HashFormatter
+        # TODO: warn when deprecated
+      end
+
+      class MessagePackFormatter < Fluent::Plugin::MessagePackFormatter
+        # TODO: warn when deprecated
+      end
+
+      class LabeledTSVFormatter < Fluent::Plugin::LabeledTSVFormatter
+        # TODO: warn when deprecated
+      end
+
+      class CsvFormatter < Fluent::Plugin::CsvFormatter
+        # TODO: warn when deprecated
+      end
+
+      class SingleValueFormatter < Fluent::Plugin::SingleValueFormatter
+        # TODO: warn when deprecated
+      end
+    end
+  end
+end