fluentd 0.12.40 → 1.6.2

data/lib/fluent/command/fluentd.rb

@@ -17,8 +17,12 @@
 require 'optparse'
 
 require 'fluent/supervisor'
+require 'fluent/log'
+require 'fluent/env'
 require 'fluent/version'
 
+$fluentdargv = Marshal.load(Marshal.dump(ARGV))
+
 op = OptionParser.new
 op.version = Fluent::VERSION
 
@@ -36,7 +40,7 @@ op.on('--dry-run', "Check fluentd setup is correct or not", TrueClass) {|b|
   opts[:dry_run] = b
 }
 
-op.on('--show-plugin-config=PLUGIN', "Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
+op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
   opts[:show_plugin_config] = plugin
 }
 
@@ -56,10 +60,19 @@ op.on('-d', '--daemon PIDFILE', "daemonize fluent process") {|s|
   opts[:daemonize] = s
 }
 
-op.on('--no-supervisor', "run without fluent supervisor") {
+op.on('--under-supervisor', "run fluent worker under supervisor (this option is NOT for users)") {
   opts[:supervise] = false
 }
 
+op.on('--no-supervisor', "run fluent worker without supervisor") {
+  opts[:supervise] = false
+  opts[:standalone_worker] = true
+}
+
+op.on('--workers NUM', "specify the number of workers under supervisor") { |i|
+  opts[:workers] = i.to_i
+}
+
 op.on('--user USER', "change user") {|s|
   opts[:chuser] = s
 }
@@ -72,7 +85,28 @@ op.on('-o', '--log PATH', "log file path") {|s|
   opts[:log_path] = s
 }
 
-op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-fly") {|s|
+ROTATE_AGE = %w(daily weekly monthly)
+op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
+  if ROTATE_AGE.include?(age)
+    opts[:log_rotate_age] = age
+  else
+    begin
+      opts[:log_rotate_age] = Integer(age)
+    rescue TypeError, ArgumentError
+      usage "log-rotate-age should be #{ROTATE_AGE.join(', ')} or a number"
+    end
+  end
+}
+
+op.on('--log-rotate-size BYTES', 'sets the byte size to rotate log files') {|s|
+  opts[:log_rotate_size] = s.to_i
+}
+
+op.on('--log-event-verbose', 'enable log events during process startup/shutdown') {|b|
+  opts[:log_event_verbose] = b
+}
+
+op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-the-fly") {|s|
   opts[:inline_config] = s
 }
 
@@ -121,7 +155,51 @@ op.on('-G', '--gem-path GEM_INSTALL_PATH', "Gemfile install path (default: $(dir
   opts[:gem_install_path] = s
 }
 
-(class << self; self; end).module_eval do
+op.on('--conf-encoding ENCODING', "specify configuration file encoding") { |s|
+  opts[:conf_encoding] = s
+}
+
+if Fluent.windows?
+  require 'windows/library'
+  include Windows::Library
+  
+  opts.merge!(
+    :winsvc_name => 'fluentdwinsvc',
+    :winsvc_display_name => 'Fluentd Windows Service',
+    :winsvc_desc => 'Fluentd is an event collector system.',
+  )
+
+  op.on('-x', '--signame INTSIGNAME', "an object name which is used for Windows Service signal (Windows only)") {|s|
+    opts[:signame] = s
+  }
+
+  op.on('--reg-winsvc MODE', "install/uninstall as Windows Service. (i: install, u: uninstall) (Windows only)") {|s|
+    opts[:regwinsvc] = s
+  }
+
+  op.on('--[no-]reg-winsvc-auto-start', "Automatically start the Windows Service at boot. (only effective with '--reg-winsvc i') (Windows only)") {|s|
+    opts[:regwinsvcautostart] = s
+  }
+
+  op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option parameters for Windows Service. (Windows only)") {|s|
+    opts[:fluentdopt] = s
+  }
+  
+  op.on('--winsvc-name NAME', "The Windows Service name to run as (Windows only)") {|s|
+    opts[:winsvc_name] = s
+  }
+  
+  op.on('--winsvc-display-name DISPLAY_NAME', "The Windows Service display name (Windows only)") {|s|
+    opts[:winsvc_display_name] = s
+  }
+  
+  op.on('--winsvc-desc DESC', "The Windows Service description (Windows only)") {|s|
+    opts[:winsvc_desc] = s
+  }
+end
+
+
+singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg
@@ -170,4 +248,74 @@ if setup_path = opts[:setup_path]
   exit 0
 end
 
-Fluent::Supervisor.new(opts).start
+early_exit = false
+start_service = false
+if winsvcinstmode = opts[:regwinsvc]
+  require 'fileutils'
+  require "win32/service"
+  require "win32/registry"
+  include Win32
+
+  case winsvcinstmode
+  when 'i'
+    binary_path = File.join(File.dirname(__FILE__), "..")
+    ruby_path = "\0" * 256
+    GetModuleFileName.call(0,ruby_path,256)
+    ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
+    start_type = Service::DEMAND_START
+    if opts[:regwinsvcautostart]
+      start_type = Service::AUTO_START
+      start_service = true
+    end
+
+    
+    Service.create(
+      service_name: opts[:winsvc_name],
+      host: nil,
+      service_type: Service::WIN32_OWN_PROCESS,
+      description: opts[:winsvc_desc],
+      start_type: start_type,
+      error_control: Service::ERROR_NORMAL,
+      binary_path_name: "\"#{ruby_path}\" -C \"#{binary_path}\"  winsvc.rb --service-name #{opts[:winsvc_name]}",
+      load_order_group: "",
+      dependencies: [""],
+      display_name: opts[:winsvc_display_name]
+    )
+  when 'u'
+    if Service.status(opts[:winsvc_name]).current_state != 'stopped'
+      begin
+        Service.stop(opts[:winsvc_name])
+      rescue => ex
+        puts "Warning: Failed to stop service: ", ex
+      end
+    end
+    Service.delete(opts[:winsvc_name])
+  else
+    # none
+  end
+  early_exit = true
+end
+
+if fluentdopt = opts[:fluentdopt]
+  Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{opts[:winsvc_name]}", Win32::Registry::KEY_ALL_ACCESS) do |reg|
+    reg['fluentdopt', Win32::Registry::REG_SZ] = fluentdopt
+  end
+  early_exit = true
+end
+
+if start_service
+  Service.start(opts[:winsvc_name])
+end
+
+exit 0 if early_exit
+
+require 'fluent/supervisor'
+if opts[:supervise]
+  Fluent::Supervisor.new(opts).run_supervisor
+else
+  if opts[:standalone_worker] && opts[:workers] && opts[:workers] > 1
+    puts "Error: multi workers is not supported with --no-supervisor"
+    exit 2
+  end
+  Fluent::Supervisor.new(opts).run_worker
+end

data/lib/fluent/command/plugin_config_formatter.rb

@@ -0,0 +1,292 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "erb"
+require "optparse"
+require "pathname"
+require "fluent/plugin"
+require "fluent/env"
+require "fluent/engine"
+require "fluent/system_config"
+require "fluent/config/element"
+require 'fluent/version'
+
+class FluentPluginConfigFormatter
+
+  AVAILABLE_FORMATS = [:markdown, :txt, :json]
+  SUPPORTED_TYPES = [
+    "input", "output", "filter",
+    "buffer", "parser", "formatter", "storage"
+  ]
+
+  DOCS_BASE_URL = "https://docs.fluentd.org/v/1.0"
+  DOCS_PLUGIN_HELPER_BASE_URL = "#{DOCS_BASE_URL}/plugin-helper-overview/"
+
+  def initialize(argv = ARGV)
+    @argv = argv
+
+    @compact = false
+    @format = :markdown
+    @verbose = false
+    @libs = []
+    @plugin_dirs = []
+    @options = {}
+
+    prepare_option_parser
+  end
+
+  def call
+    parse_options!
+    init_libraries
+    @plugin = Fluent::Plugin.__send__("new_#{@plugin_type}", @plugin_name)
+    dumped_config = {}
+    if @plugin.class.respond_to?(:plugin_helpers)
+      dumped_config[:plugin_helpers] = @plugin.class.plugin_helpers
+    end
+    @plugin.class.ancestors.reverse_each do |plugin_class|
+      next unless plugin_class.respond_to?(:dump_config_definition)
+      unless @verbose
+        next if plugin_class.name =~ /::PluginHelper::/
+      end
+      dumped_config_definition = plugin_class.dump_config_definition
+      dumped_config[plugin_class.name] = dumped_config_definition unless dumped_config_definition.empty?
+    end
+    case @format
+    when :txt
+      puts dump_txt(dumped_config)
+    when :markdown
+      puts dump_markdown(dumped_config)
+    when :json
+      puts dump_json(dumped_config)
+    end
+  end
+
+  private
+
+  def dump_txt(dumped_config)
+    dumped = ""
+    plugin_helpers = dumped_config.delete(:plugin_helpers)
+    if plugin_helpers && !plugin_helpers.empty?
+      dumped << "helpers: #{plugin_helpers.join(',')}\n"
+    end
+    if @verbose
+      dumped_config.each do |name, config|
+        dumped << "#{name}\n"
+        dumped << dump_section_txt(config)
+      end
+    else
+      configs = dumped_config.values
+      root_section = configs.shift
+      configs.each do |config|
+        root_section.update(config)
+      end
+      dumped << dump_section_txt(root_section)
+    end
+    dumped
+  end
+
+  def dump_section_txt(base_section, level = 0)
+    dumped = ""
+    indent = " " * level
+    if base_section[:section]
+      sections = []
+      params = base_section
+    else
+      sections, params = base_section.partition {|_name, value| value[:section] }
+    end
+    params.each do |name, config|
+      next if name == :section
+      dumped << "#{indent}#{name}: #{config[:type]}: (#{config[:default].inspect})"
+      dumped << " # #{config[:description]}" if config.key?(:description)
+      dumped << "\n"
+    end
+    sections.each do |section_name, sub_section|
+      required = sub_section.delete(:required)
+      multi = sub_section.delete(:multi)
+      alias_name = sub_section.delete(:alias)
+      required_label = required ? "required" : "optional"
+      multi_label = multi ? "multiple" : "single"
+      alias_label = "alias: #{alias_name}"
+      dumped << "#{indent}<#{section_name}>: #{required_label}, #{multi_label}"
+      if alias_name
+        dumped << ", #{alias_label}\n"
+      else
+        dumped << "\n"
+      end
+      sub_section.delete(:section)
+      dumped << dump_section_txt(sub_section, level + 1)
+    end
+    dumped
+  end
+
+  def dump_markdown(dumped_config)
+    dumped = ""
+    plugin_helpers = dumped_config.delete(:plugin_helpers)
+    if plugin_helpers && !plugin_helpers.empty?
+      dumped = "## Plugin helpers\n\n"
+      plugin_helpers.each do |plugin_helper|
+        dumped << "* #{plugin_helper_markdown_link(plugin_helper)}\n"
+      end
+      dumped << "\n"
+    end
+    dumped_config.each do |name, config|
+      if name == @plugin.class.name
+        dumped << "## #{name}\n\n"
+        dumped << dump_section_markdown(config)
+      else
+        dumped << "* See also: #{plugin_overview_markdown_link(name)}\n\n"
+      end
+    end
+    dumped
+  end
+
+  def dump_section_markdown(base_section, level = 0)
+    dumped = ""
+    if base_section[:section]
+      sections = []
+      params = base_section
+    else
+      sections, params = base_section.partition {|_name, value| value[:section] }
+    end
+    params.each do |name, config|
+      next if name == :section
+      template_name = @compact ? "param.md-compact.erb" : "param.md.erb"
+      template = template_path(template_name).read
+      dumped <<
+        if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+          ERB.new(template, trim_mode: "-")
+        else
+          ERB.new(template, nil, "-")
+        end.result(binding)
+    end
+    dumped << "\n"
+    sections.each do |section_name, sub_section|
+      required = sub_section.delete(:required)
+      multi = sub_section.delete(:multi)
+      alias_name = sub_section.delete(:alias)
+      sub_section.delete(:section)
+      template = template_path("section.md.erb").read
+      dumped <<
+        if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+          ERB.new(template, trim_mode: "-")
+        else
+          ERB.new(template, nil, "-")
+        end.result(binding)
+    end
+    dumped
+  end
+
+  def dump_json(dumped_config)
+    if @compact
+      JSON.generate(dumped_config)
+    else
+      JSON.pretty_generate(dumped_config)
+    end
+  end
+
+  def plugin_helper_url(plugin_helper)
+    "#{DOCS_PLUGIN_HELPER_BASE_URL}api-plugin-helper-#{plugin_helper}"
+  end
+
+  def plugin_helper_markdown_link(plugin_helper)
+    "[#{plugin_helper}](#{plugin_helper_url(plugin_helper)})"
+  end
+
+  def plugin_overview_url(class_name)
+    plugin_type = class_name.slice(/::(\w+)\z/, 1).downcase
+    "#{DOCS_BASE_URL}/#{plugin_type}#overview"
+  end
+
+  def plugin_overview_markdown_link(class_name)
+    plugin_type = class_name.slice(/::(\w+)\z/, 1)
+    "[#{plugin_type} Plugin Overview](#{plugin_overview_url(class_name)})"
+  end
+
+  def usage(message = nil)
+    puts @parser.to_s
+    puts
+    puts "Error: #{message}" if message
+    exit(false)
+  end
+
+  def prepare_option_parser
+    @parser = OptionParser.new
+    @parser.version = Fluent::VERSION
+    @parser.banner = <<BANNER
+Usage: #{$0} [options] <type> <name>
+
+Output plugin config definitions
+
+Arguments:
+\ttype: #{SUPPORTED_TYPES.join(",")}
+\tname: registered plugin name
+
+Options:
+BANNER
+    @parser.on("--verbose", "Be verbose") do
+      @verbose = true
+    end
+    @parser.on("-c", "--compact", "Compact output") do
+      @compact = true
+    end
+    @parser.on("-f", "--format=FORMAT", "Specify format. (#{AVAILABLE_FORMATS.join(',')})") do |s|
+      format = s.to_sym
+      usage("Unsupported format: #{s}") unless AVAILABLE_FORMATS.include?(format)
+      @format = format
+    end
+    @parser.on("-I PATH", "Add PATH to $LOAD_PATH") do |s|
+      $LOAD_PATH.unshift(s)
+    end
+    @parser.on("-r NAME", "Load library") do |s|
+      @libs << s
+    end
+    @parser.on("-p", "--plugin=DIR", "Add plugin directory") do |s|
+      @plugin_dirs << s
+    end
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+
+    raise "Must specify plugin type and name" unless @argv.size == 2
+
+    @plugin_type, @plugin_name = @argv
+    @options = {
+      compact: @compact,
+      format: @format,
+      verbose: @verbose,
+    }
+  rescue => e
+    usage(e)
+  end
+
+  def init_libraries
+    @libs.each do |lib|
+      require lib
+    end
+
+    @plugin_dirs.each do |dir|
+      if Dir.exist?(dir)
+        dir = File.expand_path(dir)
+        Fluent::Plugin.add_plugin_dir(dir)
+      end
+    end
+  end
+
+  def template_path(name)
+    (Pathname(__dir__) + "../../../templates/plugin_config_formatter/#{name}").realpath
+  end
+end